0
0
mirror of https://github.com/yse/easy_profiler.git synced 2024-12-26 16:11:02 +08:00

Add command line tool (profiler_converter) for converting to json format

related to #67
This commit is contained in:
derevnja 2018-02-07 19:11:12 +03:00 committed by Sergey Yagovtsev
parent d3c21bfe2a
commit 647eced7d2
10 changed files with 13768 additions and 3 deletions

View File

@ -27,6 +27,7 @@ SET(CMAKE_INSTALL_RPATH "$ORIGIN")
add_subdirectory(easy_profiler_core)
add_subdirectory(profiler_gui)
add_subdirectory(easy_profiler_converter)
if (NOT EASY_PROFILER_NO_SAMPLES)
add_subdirectory(sample)

View File

@ -0,0 +1,13 @@
set(CPP_FILES
converter.cpp
reader.cpp)
set(HEADER_FILES
converter.h
reader.h)
include_directories(../easy_profiler_core/)
include_directories(./include)
add_executable(profiler_converter ${HEADER_FILES} ${CPP_FILES} main.cpp)
target_link_libraries(profiler_converter easy_profiler)

View File

@ -0,0 +1,70 @@
///this
#include "converter.h"
/// reader
#include "reader.h"
#include <fstream>
void JSONConverter::readThreadBlocks(const profiler::reader::BlocksTreeNode &node,nlohmann::json& json)
{
auto j_local = nlohmann::json::object();
if(node.current_block != nullptr){
json = {{"id",static_cast<int>(node.current_block->blockId)}};
json["start"] = (node.current_block->beginTime);
json["stop"] = (node.current_block->endTime);
///read data from block desciptor
if(node.current_block->descriptor)
{
json["compileTimeName"] = node.current_block->descriptor->compileTimeName;
std::stringstream stream;
stream << "0x"
<< std::hex << node.current_block->descriptor->argbColor;
std::string result( stream.str() );
json["color"] = result;
json["blockType"] = node.current_block->descriptor->blockType;
}
}
auto jsonObjects = nlohmann::json::array();
for(const auto &value : node.children)
{
jsonObjects.push_back(nlohmann::json::object());
readThreadBlocks(*value.get(),jsonObjects.back());
}
json["children"] = jsonObjects;
}
void JSONConverter::convert()
{
profiler::reader::FileReader fr;
fr.readFile(m_file_in);
const profiler::reader::thread_blocks_tree_t &blocks_tree = fr.getBlocksTreeData();
nlohmann::json json;
json["version"] = fr.getVersion();
json["timeUnit"] = "ns";
auto jsonObjects = nlohmann::json::array();
for(const auto &value : blocks_tree)
{
jsonObjects.push_back(nlohmann::json::object());
jsonObjects.back()["threadId"] = value.first;
jsonObjects.back()["name"] = fr.getThreadName(value.first);
readThreadBlocks(value.second,jsonObjects.back());
}
json["threads"] = jsonObjects;
if(!m_file_out.empty())
{
std::ofstream file(m_file_out);
file << json;
}
else
{
::std::cout << nlohmann::json(json).dump(2);
}
}

View File

@ -0,0 +1,34 @@
#ifndef EASY_PROFILER_CONVERTER_H
#define EASY_PROFILER_CONVERTER_H
///std
#include<string>
///this
#include "reader.h"
///nlohmann json
#include "include/json.hpp"
class JSONConverter EASY_FINAL
{
public:
JSONConverter(const ::std::string &file_in,
const ::std::string &file_out):
m_file_in(file_in),
m_file_out(file_out)
{}
~JSONConverter()
{
}
void convert();
private:
void readThreadBlocks(const profiler::reader::BlocksTreeNode &node, nlohmann::json &json);
::std::string m_file_in;
::std::string m_file_out;
nlohmann::json json;
};
#endif //EASY_PROFILER_CONVERTER_H

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,36 @@
///std
#include <iostream>
#include <memory>
///this
//#include "reader.h"
#include "converter.h"
using namespace profiler::reader;
int main(int argc, char* argv[])
{
std::string filename;
std::string output_json_filename = "";
if (argc > 1 && argv[1])
{
filename = argv[1];
}
else
{
std::cout << "Usage: " << argv[0] << " INPUT_PROF_FILE [OUTPUT_JSON_FILE]\n"
"where:\n"
"INPUT_PROF_FILE required\n"
"OUTPUT_JSON_FILE optional (if not specified output will be print in stdout)\n";
return 1;
}
if (argc > 2 && argv[2])
{
output_json_filename = argv[2];
}
JSONConverter js(filename, output_json_filename);
js.convert();
return 0;
}

View File

@ -0,0 +1,390 @@
///std
#include <memory>
#include <algorithm>
#include <fstream>
#include <sstream>
#include <iterator>
///this
#include "reader.h"
#include "hashed_cstr.h"
////from easy_profiler_core/reader.cpp/////
typedef uint64_t processid_t;
extern const uint32_t PROFILER_SIGNATURE = ('E' << 24) | ('a' << 16) | ('s' << 8) | 'y';
# define EASY_VERSION_INT(v_major, v_minor, v_patch) ((static_cast<uint32_t>(v_major) << 24) | (static_cast<uint32_t>(v_minor) << 16) | static_cast<uint32_t>(v_patch))
const uint32_t MIN_COMPATIBLE_VERSION = EASY_VERSION_INT(0, 1, 0); ///< minimal compatible version (.prof file format was not changed seriously since this version)
const uint32_t EASY_V_100 = EASY_VERSION_INT(1, 0, 0); ///< in v1.0.0 some additional data were added into .prof file
const uint32_t EASY_V_130 = EASY_VERSION_INT(1, 3, 0); ///< in v1.3.0 changed sizeof(thread_id_t) uint32_t -> uint64_t
# undef EASY_VERSION_INT
const uint64_t TIME_FACTOR = 1000000000ULL;
// TODO: use 128 bit integer operations for better accuracy
#define EASY_USE_FLOATING_POINT_CONVERSION
#ifdef EASY_USE_FLOATING_POINT_CONVERSION
// Suppress warnings about double to uint64 conversion
# ifdef _MSC_VER
# pragma warning(disable:4244)
# elif defined(__GNUC__)
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wconversion"
# pragma GCC diagnostic ignored "-Wsign-conversion"
# elif defined(__clang__)
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wconversion"
# pragma clang diagnostic ignored "-Wsign-conversion"
# endif
# define EASY_CONVERT_TO_NANO(t, freq, factor) t *= factor
#else
# define EASY_CONVERT_TO_NANO(t, freq, factor) t *= TIME_FACTOR; t /= freq
#endif
inline bool isCompatibleVersion(uint32_t _version)
{
return _version >= MIN_COMPATIBLE_VERSION;
}
#ifdef EASY_PROFILER_HASHED_CSTR_DEFINED
typedef ::std::unordered_map<::profiler::block_id_t, ::profiler::BlockStatistics*, ::profiler::passthrough_hash<::profiler::block_id_t> > StatsMap;
/** \note It is absolutely safe to use hashed_cstr (which simply stores pointer) because std::unordered_map,
which uses it as a key, exists only inside fillTreesFromFile function. */
typedef ::std::unordered_map<::profiler::hashed_cstr, ::profiler::block_id_t> IdMap;
typedef ::std::unordered_map<::profiler::hashed_cstr, ::profiler::BlockStatistics*> CsStatsMap;
#else
// TODO: Create optimized version of profiler::hashed_cstr for Linux too.
typedef ::std::unordered_map<::profiler::block_id_t, ::profiler::BlockStatistics*, ::profiler::passthrough_hash<::profiler::block_id_t> > StatsMap;
typedef ::std::unordered_map<::profiler::hashed_stdstring, ::profiler::block_id_t> IdMap;
typedef ::std::unordered_map<::profiler::hashed_stdstring, ::profiler::BlockStatistics*> CsStatsMap;
#endif
/// end from easy_profiler_core/reader.cpp/////
using namespace profiler::reader;
::profiler::block_index_t FileReader::readFile(const ::std::string &filename)
{
::std::ifstream file(filename, ::std::fstream::binary);
if (!file.is_open())
{
errorMessage << "Can not open file " << filename;
return 0;
}
::std::stringstream inFile;
inFile << file.rdbuf();
file.close();
uint32_t signature = 0;
inFile.read((char*)&signature, sizeof(uint32_t));
if (signature != PROFILER_SIGNATURE)
{
errorMessage << "Wrong signature " << signature << "\nThis is not EasyProfiler file/stream.";
return 0;
}
m_version = 0;
inFile.read((char*)&m_version, sizeof(uint32_t));
if (!isCompatibleVersion(m_version))
{
errorMessage << "Incompatible version: v" << (m_version >> 24) << "." << ((m_version & 0x00ff0000) >> 16) << "." << (m_version & 0x0000ffff);
return 0;
}
processid_t pid = 0;
if (m_version > EASY_V_100)
{
if (m_version < EASY_V_130)
{
uint32_t old_pid = 0;
inFile.read((char*)&old_pid, sizeof(uint32_t));
pid = old_pid;
}
else
{
inFile.read((char*)&pid, sizeof(processid_t));
}
}
int64_t file_cpu_frequency = 0LL;
inFile.read((char*)&file_cpu_frequency, sizeof(int64_t));
uint64_t cpu_frequency = file_cpu_frequency;
const double conversion_factor = static_cast<double>(TIME_FACTOR) / static_cast<double>(cpu_frequency);
::profiler::timestamp_t begin_time = 0ULL;
::profiler::timestamp_t end_time = 0ULL;
inFile.read((char*)&begin_time, sizeof(::profiler::timestamp_t));
inFile.read((char*)&end_time, sizeof(::profiler::timestamp_t));
if (cpu_frequency != 0)
{
EASY_CONVERT_TO_NANO(begin_time, cpu_frequency, conversion_factor);
EASY_CONVERT_TO_NANO(end_time, cpu_frequency, conversion_factor);
}
uint32_t total_blocks_number = 0;
inFile.read((char*)&total_blocks_number, sizeof(uint32_t));
if (total_blocks_number == 0)
{
errorMessage << "Profiled blocks number == 0";
return 0;
}
uint64_t memory_size = 0;
inFile.read((char*)&memory_size, sizeof(decltype(memory_size)));
if (memory_size == 0)
{
errorMessage << "Wrong memory size == 0 for " << total_blocks_number << " blocks";
return 0;
}
uint32_t total_descriptors_number = 0;
inFile.read((char*)&total_descriptors_number, sizeof(uint32_t));
if (total_descriptors_number == 0)
{
errorMessage << "Blocks description number == 0";
return 0;
}
uint64_t descriptors_memory_size = 0;
inFile.read((char*)&descriptors_memory_size, sizeof(decltype(descriptors_memory_size)));
if (descriptors_memory_size == 0)
{
errorMessage << "Wrong memory size == 0 for " << total_descriptors_number << " blocks descriptions";
return 0;
}
m_BlockDescriptors.reserve(total_descriptors_number);
serialized_descriptors.set(descriptors_memory_size);
///read descriptors data
uint64_t i = 0;
while (!inFile.eof() && m_BlockDescriptors.size() < total_descriptors_number)
{
uint16_t sz = 0;
inFile.read((char*)&sz, sizeof(sz));
if (sz == 0)
{
m_BlockDescriptors.push_back(nullptr);
continue;
}
char* data = serialized_descriptors[i];
inFile.read(data, sz);
auto descriptor = reinterpret_cast<::profiler::SerializedBlockDescriptor*>(data);
m_BlockDescriptors.push_back(::std::make_shared<BlockDescriptor>());
m_BlockDescriptors.back()->lineNumber = descriptor->line();
m_BlockDescriptors.back()->blockId = descriptor->id();
m_BlockDescriptors.back()->argbColor = descriptor->color();
m_BlockDescriptors.back()->blockType = descriptor->type();
m_BlockDescriptors.back()->status = descriptor->status();
m_BlockDescriptors.back()->compileTimeName = descriptor->name();
m_BlockDescriptors.back()->fileName = descriptor->file();
i += sz;
}
serialized_blocks.set(memory_size);
i = 0;
uint32_t read_number = 0;
::profiler::block_index_t blocks_counter = 0;
::std::vector<char> name;
const size_t thread_id_t_size = m_version < EASY_V_130 ? sizeof(uint32_t) : sizeof(::profiler::thread_id_t);
///read blocks info for every thread
while (!inFile.eof() && read_number < total_blocks_number)
{
::profiler::thread_id_t thread_id = 0;
inFile.read((char*)&thread_id, thread_id_t_size);
auto& root = m_BlocksTree[thread_id];
uint16_t name_size = 0;
inFile.read((char*)&name_size, sizeof(uint16_t));
if (name_size != 0)
{
name.resize(name_size);
inFile.read(name.data(), name_size);
m_threadNames[thread_id] = name.data();
}
uint32_t blocks_number_in_thread = 0;
inFile.read((char*)&blocks_number_in_thread, sizeof(decltype(blocks_number_in_thread)));
auto threshold = read_number + blocks_number_in_thread;
while (!inFile.eof() && read_number < threshold)
{
++read_number;
uint16_t sz = 0;
inFile.read((char*)&sz, sizeof(sz));
if (sz == 0)
{
errorMessage << "Bad CSwitch block size == 0";
return 0;
}
char* data = serialized_blocks[i];
inFile.read(data, sz);
i += sz;
auto baseData = reinterpret_cast<::profiler::SerializedCSwitch*>(data);
auto t_begin = reinterpret_cast<::profiler::timestamp_t*>(data);
auto t_end = t_begin + 1;
if (cpu_frequency != 0)
{
EASY_CONVERT_TO_NANO(*t_begin, cpu_frequency, conversion_factor);
EASY_CONVERT_TO_NANO(*t_end, cpu_frequency, conversion_factor);
}
if (*t_end > begin_time)
{
if (*t_begin < begin_time)
*t_begin = begin_time;
m_ContextSwitches.emplace_back();
::std::shared_ptr<ContextSwitchEvent>& cs = m_ContextSwitches.back();
cs->switchName = baseData->name();
cs->targetThreadId = baseData->tid();
cs->beginTime = baseData->begin();
cs->endTime = baseData->end();
const auto block_index = blocks_counter++;
}
}
if (inFile.eof())
break;
blocks_number_in_thread = 0;
inFile.read((char*)&blocks_number_in_thread, sizeof(decltype(blocks_number_in_thread)));
threshold = read_number + blocks_number_in_thread;
::std::vector<::std::shared_ptr<BlocksTreeNode>> siblings;
::std::shared_ptr<BlocksTreeNode> prev_node = ::std::make_shared<BlocksTreeNode>();
prev_node->current_block = ::std::make_shared<BlockInfo>();
::std::shared_ptr<BlocksTreeNode> element;
uint level = 0;
while (!inFile.eof() && read_number < threshold)
{
element = ::std::make_shared<BlocksTreeNode>();
element->current_block = ::std::make_shared<BlockInfo>();
++read_number;
uint16_t sz = 0;
inFile.read((char*)&sz, sizeof(sz));
if (sz == 0)
{
errorMessage << "Bad block size == 0";
return 0;
}
char* data = serialized_blocks[i];
inFile.read(data, sz);
i += sz;
auto baseData = reinterpret_cast<::profiler::SerializedBlock*>(data);
if (baseData->id() >= total_descriptors_number)
{
errorMessage << "Bad block id == " << baseData->id();
return 0;
}
element->current_block->blockId = baseData->id();
auto desc = m_BlockDescriptors[baseData->id()];
if (desc == nullptr)
{
errorMessage << "Bad block id == " << baseData->id() << ". Description is null.";
return 0;
}
element->current_block->descriptor = m_BlockDescriptors[baseData->id()];
auto t_begin = reinterpret_cast<::profiler::timestamp_t*>(data);
auto t_end = t_begin + 1;
if (cpu_frequency != 0)
{
EASY_CONVERT_TO_NANO(*t_begin, cpu_frequency, conversion_factor);
EASY_CONVERT_TO_NANO(*t_end, cpu_frequency, conversion_factor);
}
if (*t_end >= begin_time)
{
if (*t_begin < begin_time)
*t_begin = begin_time;
element->current_block->beginTime = baseData->begin();
element->current_block->endTime = baseData->end();
///is sibling?
if(element->current_block->beginTime >= prev_node->current_block->endTime)
{
prev_node = element;
///all siblings
root.children.push_back(element);
}
else
{
auto iter = root.children.begin();
for(;iter != root.children.end(); ++iter)
{
if(iter->get()->current_block->beginTime >= element->current_block->beginTime)
{
::std::move(iter,root.children.end(),::std::back_inserter(element->children));
root.children.erase(std::remove(begin(root.children), end(root.children), nullptr),
end(root.children));
root.children.emplace_back(element);
break;
}
}
}
const auto block_index = blocks_counter++;
///TODO: make optimization BLOCK_TYPE_EVENT. leave it here commented.
// if (desc->blockType == ::profiler::BLOCK_TYPE_EVENT)
// {
// root.children.emplace_back(element);
// }
}
}
}
return blocks_counter;
}
const thread_blocks_tree_t &FileReader::getBlocksTreeData()
{
return m_BlocksTree;
}
const ::std::string &FileReader::getThreadName(uint64_t threadId)
{
return m_threadNames[threadId];
}
uint32_t FileReader::getVersion()
{
return m_version;
}
const context_switches_t &FileReader::getContextSwitches()
{
return m_ContextSwitches;
}

View File

@ -0,0 +1,98 @@
#ifndef EASY_PROFILER_READER_H
#define EASY_PROFILER_READER_H
///std
#include <fstream>
#include <vector>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
///this
#include <easy/easy_protocol.h>
#include <easy/reader.h>
#ifndef uint
#define uint unsigned int
#endif
namespace profiler{
namespace reader {
class BlocksTreeNode
{
public:
::std::shared_ptr<BlockInfo> current_block;
BlocksTreeNode* parent;
::std::vector<::std::shared_ptr<BlocksTreeNode>> children;
BlocksTreeNode(BlocksTreeNode&& other)
: current_block(::std::move(other.current_block)),
parent(other.parent),
children(::std::move(other.children))
{
}
BlocksTreeNode():current_block(nullptr),
parent(nullptr)
{
}
};
typedef ::std::unordered_map<::profiler::thread_id_t, BlocksTreeNode, ::profiler::passthrough_hash<::profiler::thread_id_t> > thread_blocks_tree_t;
typedef ::std::unordered_map<::profiler::thread_id_t, ::std::string> thread_names_t;
typedef ::std::vector<::std::shared_ptr<ContextSwitchEvent> > context_switches_t;
class FileReader EASY_FINAL
{
public:
FileReader()
{ }
~FileReader()
{ }
/*-----------------------------------------------------------------*/
///initial read file with RAW data
::profiler::block_index_t readFile(const ::std::string& filename);
/*-----------------------------------------------------------------*/
///get blocks tree
const thread_blocks_tree_t& getBlocksTreeData();
/*-----------------------------------------------------------------*/
/*! get thread name by Id
\param threadId thread Id
\return Name of thread
*/
const std::string &getThreadName(uint64_t threadId);
/*-----------------------------------------------------------------*/
/*! get file version
\return data file version
*/
uint32_t getVersion();
/*-----------------------------------------------------------------*/
///get sontext switches
const context_switches_t& getContextSwitches();
/*-----------------------------------------------------------------*/
private:
///serialized raw data
::profiler::SerializedData serialized_blocks, serialized_descriptors;
///error log stream
::std::stringstream errorMessage;
///thread's blocks
thread_blocks_tree_t m_BlocksTree;
///[thread_id, thread_name]
thread_names_t m_threadNames;
///context switches info
context_switches_t m_ContextSwitches;
std::vector<std::shared_ptr<BlockDescriptor>> m_BlockDescriptors;
///data file version
uint32_t m_version;
};
} //namespace reader
} //namespace profiler
#endif //EASY_PROFILER_READER_H

View File

@ -0,0 +1,77 @@
#ifndef EASY_PROPROTOCOL_H
#define EASY_PROPROTOCOL_H
///C++
#include <string>
#include <vector>
#include <memory>
///this
#include <easy/serialized_block.h>
//#include <easy/profiler.h>
///for actual version vistit https://github.com/yse/easy_profiler/wiki/.prof-file-format-v1.3.0
namespace profiler {
namespace reader {
struct BlockDescriptor;
struct BlocksInfo //12
{
uint32_t totalBlocksCount; //4 bytes
uint64_t totalBlocksMemory; //8 bytes
};
struct DescriptorsInfo //12
{
uint32_t allDescriptorsCount; //4 bytes
uint64_t allDescriptorsMemory; //8 bytes
};
struct FileHeader //64
{
uint32_t signature; //4
uint32_t version; //4
uint64_t processId; //8
int64_t cpuFrequency; //8
uint64_t beginTime; //8
uint64_t endTime; //8
BlocksInfo serializedBlocksInfo; //12
DescriptorsInfo blocksDescriptorInfo; //12
};
struct BlockInfo
{
uint64_t beginTime;
uint64_t endTime;
uint32_t blockId;
uint32_t parentBlockId;
std::string runTimeBlockName;
std::string thread_name; ///< Name of parent thread
std::shared_ptr<BlockDescriptor> descriptor;
};
struct ContextSwitchEvent{
uint64_t beginTime;
uint64_t endTime;
uint64_t targetThreadId;
std::string switchName;
};
struct BlockDescriptor
{
uint32_t blockId;
int lineNumber;
uint32_t argbColor;
uint8_t blockType;
uint8_t status;
std::string compileTimeName;
std::string fileName;
};
} //namespace reader
} //namespace profiler
#endif

View File

@ -1,6 +1,6 @@
/************************************************************************
* file name : reader.cpp
* ----------------- :
* ----------------- :
* creation time : 2016/06/19
* authors : Sergey Yagovtsev, Victor Zarubkin
* emails : yse.sey@gmail.com, v.s.zarubkin@gmail.com
@ -23,7 +23,7 @@
* : * 2016/06/30 Victor Zarubkin: Added this header.
* : Added tree depth calculation.
* :
* : *
* : *
* ----------------- :
* license : Lightweight profiler library for c++
* : Copyright(C) 2016-2017 Sergey Yagovtsev, Victor Zarubkin
@ -70,6 +70,7 @@
#include "hashed_cstr.h"
#include <iostream>
#include <fstream>
#include <sstream>
#include <iterator>
@ -419,7 +420,7 @@ extern "C" {
typedef ::std::basic_iostream<::std::stringstream::char_type, ::std::stringstream::traits_type> stringstream_parent;
stringstream_parent& s = str;
auto oldbuf = s.rdbuf(inFile.rdbuf());
// Read data from file
auto result = fillTreesFromStream(progress, str, serialized_blocks, serialized_descriptors, descriptors, blocks,
threaded_trees, total_descriptors_number, version, gather_statistics, _log);