From 84b391b6cc84eaa9d9d823d9836bb97d787111d9 Mon Sep 17 00:00:00 2001 From: Victor Zarubkin Date: Tue, 8 May 2018 21:14:46 +0300 Subject: [PATCH] #0 [Core] refactoring --- easy_profiler_converter/reader.h | 6 +- easy_profiler_core/CMakeLists.txt | 2 + easy_profiler_core/include/easy/reader.h | 59 +-- easy_profiler_core/include/easy/writer.h | 105 +++++ easy_profiler_core/reader.cpp | 412 -------------------- easy_profiler_core/writer.cpp | 466 +++++++++++++++++++++++ profiler_gui/main_window.cpp | 1 + 7 files changed, 580 insertions(+), 471 deletions(-) create mode 100644 easy_profiler_core/include/easy/writer.h create mode 100644 easy_profiler_core/writer.cpp diff --git a/easy_profiler_converter/reader.h b/easy_profiler_converter/reader.h index 28ee357..1ce825a 100644 --- a/easy_profiler_converter/reader.h +++ b/easy_profiler_converter/reader.h @@ -40,8 +40,8 @@ The Apache License, Version 2.0 (the "License"); **/ -#ifndef EASY_PROFILER_READER_H -#define EASY_PROFILER_READER_H +#ifndef EASY_PROFILER_CONVERTER_READER_H +#define EASY_PROFILER_CONVERTER_READER_H #include #include @@ -140,4 +140,4 @@ private: } // end of namespace profiler. -#endif // EASY_PROFILER_READER_H +#endif // EASY_PROFILER_CONVERTER_READER_H diff --git a/easy_profiler_core/CMakeLists.txt b/easy_profiler_core/CMakeLists.txt index 8b4c97c..7d5bec4 100644 --- a/easy_profiler_core/CMakeLists.txt +++ b/easy_profiler_core/CMakeLists.txt @@ -123,6 +123,7 @@ set(CPP_FILES reader.cpp serialized_block.cpp thread_storage.cpp + writer.cpp ) set(H_FILES @@ -148,6 +149,7 @@ set(INCLUDE_FILES ${EASY_INCLUDE_DIR}/reader.h ${EASY_INCLUDE_DIR}/utility.h ${EASY_INCLUDE_DIR}/serialized_block.h + ${EASY_INCLUDE_DIR}/writer.h ${EASY_INCLUDE_DIR}/details/arbitrary_value_aux.h ${EASY_INCLUDE_DIR}/details/arbitrary_value_public_types.h ${EASY_INCLUDE_DIR}/details/easy_compiler_support.h diff --git a/easy_profiler_core/include/easy/reader.h b/easy_profiler_core/include/easy/reader.h index aac2322..d2694b8 100644 --- a/easy_profiler_core/include/easy/reader.h +++ b/easy_profiler_core/include/easy/reader.h @@ -40,8 +40,8 @@ The Apache License, Version 2.0 (the "License"); **/ -#ifndef PROFILER_READER____H -#define PROFILER_READER____H +#ifndef EASY_PROFILER_READER_H +#define EASY_PROFILER_READER_H ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -362,27 +362,6 @@ extern "C" { profiler::descriptors_list_t& descriptors, std::ostream& _log); - PROFILER_API profiler::block_index_t writeTreesToFile(std::atomic& progress, const char* filename, - const profiler::SerializedData& serialized_descriptors, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count, - const profiler::thread_blocks_tree_t& trees, - profiler::block_getter_fn block_getter, - profiler::timestamp_t begin_time, - profiler::timestamp_t end_time, - profiler::processid_t pid, - std::ostream& log); - - PROFILER_API profiler::block_index_t writeTreesToStream(std::atomic& progress, std::ostream& str, - const profiler::SerializedData& serialized_descriptors, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count, - const profiler::thread_blocks_tree_t& trees, - profiler::block_getter_fn block_getter, - profiler::timestamp_t begin_time, - profiler::timestamp_t end_time, - profiler::processid_t pid, - std::ostream& log); } inline profiler::block_index_t fillTreesFromFile(const char* filename, profiler::SerializedData& serialized_blocks, @@ -400,38 +379,6 @@ inline profiler::block_index_t fillTreesFromFile(const char* filename, profiler: threaded_trees, total_descriptors_number, version, pid, gather_statistics, _log); } -inline profiler::block_index_t writeTreesToFile(const char* filename, - const profiler::SerializedData& serialized_descriptors, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count, - const profiler::thread_blocks_tree_t& trees, - profiler::block_getter_fn block_getter, - profiler::timestamp_t begin_time, - profiler::timestamp_t end_time, - profiler::processid_t pid, - std::ostream& log) -{ - std::atomic progress = ATOMIC_VAR_INIT(0); - return writeTreesToFile(progress, filename, serialized_descriptors, descriptors, descriptors_count, trees, - std::move(block_getter), begin_time, end_time, pid, log); -} - -inline profiler::block_index_t writeTreesToStream(std::ostream& str, - const profiler::SerializedData& serialized_descriptors, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count, - const profiler::thread_blocks_tree_t& trees, - profiler::block_getter_fn block_getter, - profiler::timestamp_t begin_time, - profiler::timestamp_t end_time, - profiler::processid_t pid, - std::ostream& log) -{ - std::atomic progress = ATOMIC_VAR_INIT(0); - return writeTreesToStream(progress, str, serialized_descriptors, descriptors, descriptors_count, trees, - std::move(block_getter), begin_time, end_time, pid, log); -} - inline bool readDescriptionsFromStream(std::istream& str, profiler::SerializedData& serialized_descriptors, profiler::descriptors_list_t& descriptors, @@ -443,4 +390,4 @@ inline bool readDescriptionsFromStream(std::istream& str, ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#endif // PROFILER_READER____H +#endif // EASY_PROFILER_READER_H diff --git a/easy_profiler_core/include/easy/writer.h b/easy_profiler_core/include/easy/writer.h new file mode 100644 index 0000000..c750fad --- /dev/null +++ b/easy_profiler_core/include/easy/writer.h @@ -0,0 +1,105 @@ +/** +Lightweight profiler library for c++ +Copyright(C) 2016-2018 Sergey Yagovtsev, Victor Zarubkin + +Licensed under either of + * MIT license (LICENSE.MIT or http://opensource.org/licenses/MIT) + * Apache License, Version 2.0, (LICENSE.APACHE or http://www.apache.org/licenses/LICENSE-2.0) +at your option. + +The MIT License + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + of the Software, and to permit persons to whom the Software is furnished + to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + +The Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +**/ + +#ifndef EASY_PROFILER_WRITER_H +#define EASY_PROFILER_WRITER_H + +#include + +extern "C" { + + PROFILER_API profiler::block_index_t writeTreesToFile(std::atomic& progress, const char* filename, + const profiler::SerializedData& serialized_descriptors, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count, + const profiler::thread_blocks_tree_t& trees, + profiler::block_getter_fn block_getter, + profiler::timestamp_t begin_time, + profiler::timestamp_t end_time, + profiler::processid_t pid, + std::ostream& log); + + PROFILER_API profiler::block_index_t writeTreesToStream(std::atomic& progress, std::ostream& str, + const profiler::SerializedData& serialized_descriptors, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count, + const profiler::thread_blocks_tree_t& trees, + profiler::block_getter_fn block_getter, + profiler::timestamp_t begin_time, + profiler::timestamp_t end_time, + profiler::processid_t pid, + std::ostream& log); +} + +inline profiler::block_index_t writeTreesToFile(const char* filename, + const profiler::SerializedData& serialized_descriptors, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count, + const profiler::thread_blocks_tree_t& trees, + profiler::block_getter_fn block_getter, + profiler::timestamp_t begin_time, + profiler::timestamp_t end_time, + profiler::processid_t pid, + std::ostream& log) +{ + std::atomic progress = ATOMIC_VAR_INIT(0); + return writeTreesToFile(progress, filename, serialized_descriptors, descriptors, descriptors_count, trees, + std::move(block_getter), begin_time, end_time, pid, log); +} + +inline profiler::block_index_t writeTreesToStream(std::ostream& str, + const profiler::SerializedData& serialized_descriptors, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count, + const profiler::thread_blocks_tree_t& trees, + profiler::block_getter_fn block_getter, + profiler::timestamp_t begin_time, + profiler::timestamp_t end_time, + profiler::processid_t pid, + std::ostream& log) +{ + std::atomic progress = ATOMIC_VAR_INIT(0); + return writeTreesToStream(progress, str, serialized_descriptors, descriptors, descriptors_count, trees, + std::move(block_getter), begin_time, end_time, pid, log); +} + +#endif //EASY_PROFILER_WRITER_H diff --git a/easy_profiler_core/reader.cpp b/easy_profiler_core/reader.cpp index b21a0f2..a91eb54 100644 --- a/easy_profiler_core/reader.cpp +++ b/easy_profiler_core/reader.cpp @@ -8,23 +8,6 @@ * description : The file contains implementation of fillTreesFromFile function * : which reads profiler file and fill profiler blocks tree. * ----------------- : -* change log : * 2016/06/19 Sergey Yagovtsev: First fillTreesFromFile implementation. -* : -* : * 2016/06/25 Victor Zarubkin: Removed unnecessary memory allocation and copy -* : when creating and inserting blocks into the tree. -* : -* : * 2016/06/26 Victor Zarubkin: Added statistics gathering (min, max, average duration, -* : number of block calls). -* : * 2016/06/26 Victor Zarubkin, Sergey Yagovtsev: Added statistics gathering for root -* : blocks in the tree. -* : -* : * 2016/06/29 Victor Zarubkin: Added calculaton of total children number for blocks. -* : -* : * 2016/06/30 Victor Zarubkin: Added this header. -* : Added tree depth calculation. -* : -* : * -* ----------------- : * license : Lightweight profiler library for c++ * : Copyright(C) 2016-2018 Sergey Yagovtsev, Victor Zarubkin * : @@ -76,7 +59,6 @@ #include #include "hashed_cstr.h" -#include "alignment_helpers.h" ////////////////////////////////////////////////////////////////////////// @@ -399,18 +381,6 @@ static bool update_progress(std::atomic& progress, int new_value, std::ostr return true; } -static bool update_progress_write(std::atomic& progress, int new_value, std::ostream& _log) -{ - auto oldprogress = progress.exchange(new_value, std::memory_order_release); - if (oldprogress < 0) - { - _log << "Writing was interrupted"; - return false; - } - - return true; -} - ////////////////////////////////////////////////////////////////////////// struct EasyFileHeader @@ -1119,388 +1089,6 @@ extern "C" PROFILER_API bool readDescriptionsFromStream(std::atomic& progre return !descriptors.empty(); } -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////// - -struct BlocksRange -{ - profiler::block_index_t begin; - profiler::block_index_t end; - - BlocksRange(profiler::block_index_t size = 0) - : begin(size), end(size) - { - } - - BlocksRange(profiler::block_index_t beginIndex, profiler::block_index_t endIndex) - : begin(beginIndex), end(endIndex) - { - } -}; - -struct BlocksMemoryAndCount -{ - uint64_t usedMemorySize = 0; // memory size used by profiler blocks - profiler::block_index_t blocksCount = 0; - - BlocksMemoryAndCount() = default; - - BlocksMemoryAndCount& operator += (const BlocksMemoryAndCount& another) - { - usedMemorySize += another.usedMemorySize; - blocksCount += another.blocksCount; - return *this; - } -}; - -struct BlocksAndCSwitchesRange -{ - BlocksMemoryAndCount blocksMemoryAndCount; - BlocksMemoryAndCount cswitchesMemoryAndCount; - BlocksRange blocks; - BlocksRange cswitches; -}; - -template -static void write(std::ostream& _stream, const char* _data, T _size) -{ - _stream.write(_data, _size); -} - -template -static void write(std::ostream& _stream, const T& _data) -{ - _stream.write((const char*)&_data, sizeof(T)); -} - -static BlocksRange findRange(const profiler::BlocksTree::children_t& children, profiler::timestamp_t beginTime, - profiler::timestamp_t endTime, const profiler::block_getter_fn& getter) -{ - const auto size = static_cast(children.size()); - BlocksRange range(size); - - auto first_it = std::lower_bound(children.begin(), children.end(), beginTime, [&](profiler::block_index_t element, profiler::timestamp_t value) - { - return getter(element).node->end() < value; - }); - - for (auto it = first_it; it != children.end(); ++it) - { - const auto& child = getter(*it); - if (child.node->begin() >= beginTime || child.node->end() > beginTime) - break; - } - - if (first_it != children.end() && getter(*first_it).node->begin() <= endTime) - { - auto last_it = std::lower_bound(children.begin(), children.end(), endTime, [&](profiler::block_index_t element, profiler::timestamp_t value) - { - return getter(element).node->begin() <= value; - }); - - if (last_it != children.end() && getter(*last_it).node->end() >= beginTime) - { - const auto begin = static_cast(std::distance(children.begin(), first_it)); - const auto end = static_cast(std::distance(children.begin(), last_it)); - - if (begin <= end) - { - range.begin = begin; - range.end = end; - } - } - } - - return range; -} - -static BlocksMemoryAndCount calculateUsedMemoryAndBlocksCount(const profiler::BlocksTree::children_t& children, - const BlocksRange& range, - const profiler::block_getter_fn& getter, - const profiler::descriptors_list_t& descriptors, - bool contextSwitches) -{ - BlocksMemoryAndCount memoryAndCount; - - if (!contextSwitches) - { - for (auto i = range.begin; i < range.end; ++i) - { - const auto& child = getter(children[i]); - - // Calculate self memory consumption - const auto& desc = *descriptors[child.node->id()]; - uint64_t usedMemorySize = 0; - - if (desc.type() == profiler::BlockType::Value) - usedMemorySize = sizeof(profiler::ArbitraryValue) + child.value->data_size(); - else - usedMemorySize = sizeof(profiler::SerializedBlock) + strlen(child.node->name()) + 1; - - // Calculate children memory consumption - const BlocksRange childRange(0, static_cast(child.children.size())); - const auto childrenMemoryAndCount = calculateUsedMemoryAndBlocksCount(child.children, childRange, - getter, descriptors, - false); - - // Accumulate memory and count - memoryAndCount += childrenMemoryAndCount; - memoryAndCount.usedMemorySize += usedMemorySize; - ++memoryAndCount.blocksCount; - } - } - else - { - for (auto i = range.begin; i < range.end; ++i) - { - const auto& child = getter(children[i]); - const uint64_t usedMemorySize = sizeof(profiler::SerializedCSwitch) + strlen(child.cs->name()) + 1; - memoryAndCount.usedMemorySize += usedMemorySize; - ++memoryAndCount.blocksCount; - } - } - - return memoryAndCount; -} - -static void serializeBlocks(std::ostream& output, std::vector& buffer, - const profiler::BlocksTree::children_t& children, const BlocksRange& range, - const profiler::block_getter_fn& getter, const profiler::descriptors_list_t& descriptors) -{ - for (auto i = range.begin; i < range.end; ++i) - { - const auto& child = getter(children[i]); - - // Serialize children - const BlocksRange childRange(0, static_cast(child.children.size())); - serializeBlocks(output, buffer, child.children, childRange, getter, descriptors); - - // Serialize self - const auto& desc = *descriptors[child.node->id()]; - uint16_t usedMemorySize = 0; - - if (desc.type() == profiler::BlockType::Value) - { - usedMemorySize = static_cast(sizeof(profiler::ArbitraryValue)) + child.value->data_size(); - buffer.resize(usedMemorySize + sizeof(uint16_t)); - unaligned_store16(buffer.data(), usedMemorySize); - memcpy(buffer.data() + sizeof(uint16_t), child.value, static_cast(usedMemorySize)); - } - else - { - usedMemorySize = static_cast(sizeof(profiler::SerializedBlock) - + strlen(child.node->name()) + 1); - - buffer.resize(usedMemorySize + sizeof(uint16_t)); - unaligned_store16(buffer.data(), usedMemorySize); - memcpy(buffer.data() + sizeof(uint16_t), child.node, static_cast(usedMemorySize)); - - if (child.node->id() != desc.id()) - { - // This block id is dynamic. Restore it's value like it was before in the input .prof file - auto block = reinterpret_cast(buffer.data() + sizeof(uint16_t)); - block->setId(desc.id()); - } - } - - write(output, buffer.data(), buffer.size()); - } -} - -static void serializeContextSwitches(std::ostream& output, std::vector& buffer, - const profiler::BlocksTree::children_t& children, const BlocksRange& range, - const profiler::block_getter_fn& getter) -{ - for (auto i = range.begin; i < range.end; ++i) - { - const auto& child = getter(children[i]); - - const auto usedMemorySize = static_cast( - sizeof(profiler::SerializedCSwitch) + strlen(child.cs->name()) + 1); - - buffer.resize(usedMemorySize + sizeof(uint16_t)); - unaligned_store16(buffer.data(), usedMemorySize); - memcpy(buffer.data() + sizeof(uint16_t), child.cs, static_cast(usedMemorySize)); - - write(output, buffer.data(), buffer.size()); - } -} - -static void serializeDescriptors(std::ostream& output, std::vector& buffer, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count) -{ - const size_t size = std::min(descriptors.size(), static_cast(descriptors_count)); - for (size_t i = 0; i < size; ++i) - { - const auto& desc = *descriptors[i]; - if (desc.id() != i) - break; - - const auto usedMemorySize = static_cast(sizeof(profiler::SerializedBlockDescriptor) - + strlen(desc.name()) + strlen(desc.file()) + 2); - - buffer.resize(usedMemorySize + sizeof(uint16_t)); - unaligned_store16(buffer.data(), usedMemorySize); - memcpy(buffer.data() + sizeof(uint16_t), &desc, static_cast(usedMemorySize)); - - write(output, buffer.data(), buffer.size()); - } -} - -extern "C" PROFILER_API profiler::block_index_t writeTreesToFile(std::atomic& progress, const char* filename, - const profiler::SerializedData& serialized_descriptors, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count, - const profiler::thread_blocks_tree_t& trees, - profiler::block_getter_fn block_getter, - profiler::timestamp_t begin_time, - profiler::timestamp_t end_time, - profiler::processid_t pid, - std::ostream& log) -{ - if (!update_progress_write(progress, 0, log)) - return 0; - - std::ofstream outFile(filename, std::fstream::binary); - if (!outFile.is_open()) - { - log << "Can not open file " << filename; - return 0; - } - - // Write data to file - auto result = writeTreesToStream(progress, outFile, serialized_descriptors, descriptors, descriptors_count, trees, - std::move(block_getter), begin_time, end_time, pid, log); - - return result; -} - -////////////////////////////////////////////////////////////////////////// - -extern "C" PROFILER_API profiler::block_index_t writeTreesToStream(std::atomic& progress, std::ostream& str, - const profiler::SerializedData& serialized_descriptors, - const profiler::descriptors_list_t& descriptors, - profiler::block_id_t descriptors_count, - const profiler::thread_blocks_tree_t& trees, - profiler::block_getter_fn block_getter, - profiler::timestamp_t begin_time, - profiler::timestamp_t end_time, - profiler::processid_t pid, - std::ostream& log) -{ - if (trees.empty() || serialized_descriptors.empty() || descriptors_count == 0) - { - log << "Nothing to save"; - return 0; - } - - BlocksMemoryAndCount total; - - using ranges_t = std::unordered_map >; - ranges_t block_ranges; - - // Calculate block ranges and used memory (for serialization) - profiler::timestamp_t beginTime = begin_time, endTime = end_time; - size_t i = 0; - for (const auto& kv : trees) - { - const auto id = kv.first; - const auto& tree = kv.second; - - BlocksAndCSwitchesRange range; - - range.blocks = findRange(tree.children, begin_time, end_time, block_getter); - range.cswitches = findRange(tree.sync, begin_time, end_time, block_getter); - - range.blocksMemoryAndCount = calculateUsedMemoryAndBlocksCount(tree.children, range.blocks, block_getter, - descriptors, false); - total += range.blocksMemoryAndCount; - - if (range.blocksMemoryAndCount.blocksCount != 0) - { - beginTime = std::min(beginTime, block_getter(tree.children[range.blocks.begin]).node->begin()); - endTime = std::max(endTime, block_getter(tree.children[range.blocks.end - 1]).node->end()); - } - - range.cswitchesMemoryAndCount = calculateUsedMemoryAndBlocksCount(tree.sync, range.cswitches, block_getter, - descriptors, true); - total += range.cswitchesMemoryAndCount; - - if (range.cswitchesMemoryAndCount.blocksCount != 0) - { - beginTime = std::min(beginTime, block_getter(tree.children[range.cswitches.begin]).cs->begin()); - endTime = std::max(endTime, block_getter(tree.children[range.cswitches.end - 1]).cs->end()); - } - - block_ranges[id] = range; - - if (!update_progress_write(progress, 15 / static_cast(trees.size() - i), log)) - return 0; - - ++i; - } - - if (total.blocksCount == 0) - { - log << "Nothing to save"; - return 0; - } - - const uint64_t usedMemorySizeDescriptors = serialized_descriptors.size() + descriptors_count * sizeof(uint16_t); - - // Write data to stream - write(str, EASY_PROFILER_SIGNATURE); - write(str, EASY_PROFILER_VERSION); - write(str, pid); - - // write 0 because we do not need to convert time from ticks to nanoseconds (it's already converted) - write(str, 0LL); // CPU frequency - - write(str, beginTime); - write(str, endTime); - - write(str, total.usedMemorySize); - write(str, usedMemorySizeDescriptors); - write(str, total.blocksCount); - write(str, descriptors_count); - - std::vector buffer; - - // Serialize all descriptors - serializeDescriptors(str, buffer, descriptors, descriptors_count); - - // Serialize all blocks - i = 0; - for (const auto& kv : trees) - { - const auto id = kv.first; - const auto& tree = kv.second; - const auto& range = block_ranges.at(id); - - const auto nameSize = static_cast(tree.thread_name.size() + 1); - write(str, id); - write(str, nameSize); - write(str, tree.name(), nameSize); - - // Serialize context switches - write(str, range.cswitchesMemoryAndCount.blocksCount); - if (range.cswitchesMemoryAndCount.blocksCount != 0) - serializeContextSwitches(str, buffer, tree.sync, range.cswitches, block_getter); - - // Serialize blocks - write(str, range.blocksMemoryAndCount.blocksCount); - if (range.blocksMemoryAndCount.blocksCount != 0) - serializeBlocks(str, buffer, tree.children, range.blocks, block_getter, descriptors); - - if (!update_progress_write(progress, 40 + 60 / static_cast(trees.size() - i), log)) - return 0; - } - - return total.blocksCount; -} - ////////////////////////////////////////////////////////////////////////// #undef EASY_CONVERT_TO_NANO diff --git a/easy_profiler_core/writer.cpp b/easy_profiler_core/writer.cpp new file mode 100644 index 0000000..b85fe4a --- /dev/null +++ b/easy_profiler_core/writer.cpp @@ -0,0 +1,466 @@ +/************************************************************************ +* file name : writer.cpp +* ----------------- : +* creation time : 2018/05/08 +* authors : Sergey Yagovtsev, Victor Zarubkin +* emails : yse.sey@gmail.com, v.s.zarubkin@gmail.com +* ----------------- : +* description : The file contains implementation of fillTreesFromFile function +* : which reads profiler file and fill profiler blocks tree. +* ----------------- : +* license : Lightweight profiler library for c++ +* : Copyright(C) 2016-2018 Sergey Yagovtsev, Victor Zarubkin +* : +* : Licensed under either of +* : * MIT license (LICENSE.MIT or http://opensource.org/licenses/MIT) +* : * Apache License, Version 2.0, (LICENSE.APACHE or http://www.apache.org/licenses/LICENSE-2.0) +* : at your option. +* : +* : The MIT License +* : +* : Permission is hereby granted, free of charge, to any person obtaining a copy +* : of this software and associated documentation files (the "Software"), to deal +* : in the Software without restriction, including without limitation the rights +* : to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +* : of the Software, and to permit persons to whom the Software is furnished +* : to do so, subject to the following conditions: +* : +* : The above copyright notice and this permission notice shall be included in all +* : copies or substantial portions of the Software. +* : +* : THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +* : INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +* : PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +* : LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* : TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +* : USE OR OTHER DEALINGS IN THE SOFTWARE. +* : +* : The Apache License, Version 2.0 (the "License") +* : +* : You may not use this file except in compliance with the License. +* : You may obtain a copy of the License at +* : +* : http://www.apache.org/licenses/LICENSE-2.0 +* : +* : Unless required by applicable law or agreed to in writing, software +* : distributed under the License is distributed on an "AS IS" BASIS, +* : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* : See the License for the specific language governing permissions and +* : limitations under the License. +************************************************************************/ + +#include +#include +#include + +#include +#include + +#include "alignment_helpers.h" + +////////////////////////////////////////////////////////////////////////// + +extern const uint32_t EASY_PROFILER_SIGNATURE; +extern const uint32_t EASY_PROFILER_VERSION; + +////////////////////////////////////////////////////////////////////////// + +struct BlocksRange +{ + profiler::block_index_t begin; + profiler::block_index_t end; + + BlocksRange(profiler::block_index_t size = 0) + : begin(size), end(size) + { + } + + BlocksRange(profiler::block_index_t beginIndex, profiler::block_index_t endIndex) + : begin(beginIndex), end(endIndex) + { + } +}; + +struct BlocksMemoryAndCount +{ + uint64_t usedMemorySize = 0; // memory size used by profiler blocks + profiler::block_index_t blocksCount = 0; + + BlocksMemoryAndCount() = default; + + BlocksMemoryAndCount& operator += (const BlocksMemoryAndCount& another) + { + usedMemorySize += another.usedMemorySize; + blocksCount += another.blocksCount; + return *this; + } +}; + +struct BlocksAndCSwitchesRange +{ + BlocksMemoryAndCount blocksMemoryAndCount; + BlocksMemoryAndCount cswitchesMemoryAndCount; + BlocksRange blocks; + BlocksRange cswitches; +}; + +////////////////////////////////////////////////////////////////////////// + +template +static void write(std::ostream& _stream, const char* _data, T _size) +{ + _stream.write(_data, _size); +} + +template +static void write(std::ostream& _stream, const T& _data) +{ + _stream.write((const char*)&_data, sizeof(T)); +} + +static bool update_progress_write(std::atomic& progress, int new_value, std::ostream& _log) +{ + auto oldprogress = progress.exchange(new_value, std::memory_order_release); + if (oldprogress < 0) + { + _log << "Writing was interrupted"; + return false; + } + + return true; +} + +////////////////////////////////////////////////////////////////////////// + +static BlocksRange findRange(const profiler::BlocksTree::children_t& children, profiler::timestamp_t beginTime, + profiler::timestamp_t endTime, const profiler::block_getter_fn& getter) +{ + const auto size = static_cast(children.size()); + BlocksRange range(size); + + auto first_it = std::lower_bound(children.begin(), children.end(), beginTime, [&](profiler::block_index_t element, profiler::timestamp_t value) + { + return getter(element).node->end() < value; + }); + + for (auto it = first_it; it != children.end(); ++it) + { + const auto& child = getter(*it); + if (child.node->begin() >= beginTime || child.node->end() > beginTime) + break; + } + + if (first_it != children.end() && getter(*first_it).node->begin() <= endTime) + { + auto last_it = std::lower_bound(children.begin(), children.end(), endTime, [&](profiler::block_index_t element, profiler::timestamp_t value) + { + return getter(element).node->begin() <= value; + }); + + if (last_it != children.end() && getter(*last_it).node->end() >= beginTime) + { + const auto begin = static_cast(std::distance(children.begin(), first_it)); + const auto end = static_cast(std::distance(children.begin(), last_it)); + + if (begin <= end) + { + range.begin = begin; + range.end = end; + } + } + } + + return range; +} + +static BlocksMemoryAndCount calculateUsedMemoryAndBlocksCount(const profiler::BlocksTree::children_t& children, + const BlocksRange& range, + const profiler::block_getter_fn& getter, + const profiler::descriptors_list_t& descriptors, + bool contextSwitches) +{ + BlocksMemoryAndCount memoryAndCount; + + if (!contextSwitches) + { + for (auto i = range.begin; i < range.end; ++i) + { + const auto& child = getter(children[i]); + + // Calculate self memory consumption + const auto& desc = *descriptors[child.node->id()]; + uint64_t usedMemorySize = 0; + + if (desc.type() == profiler::BlockType::Value) + usedMemorySize = sizeof(profiler::ArbitraryValue) + child.value->data_size(); + else + usedMemorySize = sizeof(profiler::SerializedBlock) + strlen(child.node->name()) + 1; + + // Calculate children memory consumption + const BlocksRange childRange(0, static_cast(child.children.size())); + const auto childrenMemoryAndCount = calculateUsedMemoryAndBlocksCount(child.children, childRange, + getter, descriptors, + false); + + // Accumulate memory and count + memoryAndCount += childrenMemoryAndCount; + memoryAndCount.usedMemorySize += usedMemorySize; + ++memoryAndCount.blocksCount; + } + } + else + { + for (auto i = range.begin; i < range.end; ++i) + { + const auto& child = getter(children[i]); + const uint64_t usedMemorySize = sizeof(profiler::SerializedCSwitch) + strlen(child.cs->name()) + 1; + memoryAndCount.usedMemorySize += usedMemorySize; + ++memoryAndCount.blocksCount; + } + } + + return memoryAndCount; +} + +////////////////////////////////////////////////////////////////////////// + +static void serializeBlocks(std::ostream& output, std::vector& buffer, + const profiler::BlocksTree::children_t& children, const BlocksRange& range, + const profiler::block_getter_fn& getter, const profiler::descriptors_list_t& descriptors) +{ + for (auto i = range.begin; i < range.end; ++i) + { + const auto& child = getter(children[i]); + + // Serialize children + const BlocksRange childRange(0, static_cast(child.children.size())); + serializeBlocks(output, buffer, child.children, childRange, getter, descriptors); + + // Serialize self + const auto& desc = *descriptors[child.node->id()]; + uint16_t usedMemorySize = 0; + + if (desc.type() == profiler::BlockType::Value) + { + usedMemorySize = static_cast(sizeof(profiler::ArbitraryValue)) + child.value->data_size(); + buffer.resize(usedMemorySize + sizeof(uint16_t)); + unaligned_store16(buffer.data(), usedMemorySize); + memcpy(buffer.data() + sizeof(uint16_t), child.value, static_cast(usedMemorySize)); + } + else + { + usedMemorySize = static_cast(sizeof(profiler::SerializedBlock) + + strlen(child.node->name()) + 1); + + buffer.resize(usedMemorySize + sizeof(uint16_t)); + unaligned_store16(buffer.data(), usedMemorySize); + memcpy(buffer.data() + sizeof(uint16_t), child.node, static_cast(usedMemorySize)); + + if (child.node->id() != desc.id()) + { + // This block id is dynamic. Restore it's value like it was before in the input .prof file + auto block = reinterpret_cast(buffer.data() + sizeof(uint16_t)); + block->setId(desc.id()); + } + } + + write(output, buffer.data(), buffer.size()); + } +} + +static void serializeContextSwitches(std::ostream& output, std::vector& buffer, + const profiler::BlocksTree::children_t& children, const BlocksRange& range, + const profiler::block_getter_fn& getter) +{ + for (auto i = range.begin; i < range.end; ++i) + { + const auto& child = getter(children[i]); + + const auto usedMemorySize = static_cast( + sizeof(profiler::SerializedCSwitch) + strlen(child.cs->name()) + 1); + + buffer.resize(usedMemorySize + sizeof(uint16_t)); + unaligned_store16(buffer.data(), usedMemorySize); + memcpy(buffer.data() + sizeof(uint16_t), child.cs, static_cast(usedMemorySize)); + + write(output, buffer.data(), buffer.size()); + } +} + +static void serializeDescriptors(std::ostream& output, std::vector& buffer, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count) +{ + const size_t size = std::min(descriptors.size(), static_cast(descriptors_count)); + for (size_t i = 0; i < size; ++i) + { + const auto& desc = *descriptors[i]; + if (desc.id() != i) + break; + + const auto usedMemorySize = static_cast(sizeof(profiler::SerializedBlockDescriptor) + + strlen(desc.name()) + strlen(desc.file()) + 2); + + buffer.resize(usedMemorySize + sizeof(uint16_t)); + unaligned_store16(buffer.data(), usedMemorySize); + memcpy(buffer.data() + sizeof(uint16_t), &desc, static_cast(usedMemorySize)); + + write(output, buffer.data(), buffer.size()); + } +} + +////////////////////////////////////////////////////////////////////////// + +extern "C" PROFILER_API profiler::block_index_t writeTreesToFile(std::atomic& progress, const char* filename, + const profiler::SerializedData& serialized_descriptors, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count, + const profiler::thread_blocks_tree_t& trees, + profiler::block_getter_fn block_getter, + profiler::timestamp_t begin_time, + profiler::timestamp_t end_time, + profiler::processid_t pid, + std::ostream& log) +{ + if (!update_progress_write(progress, 0, log)) + return 0; + + std::ofstream outFile(filename, std::fstream::binary); + if (!outFile.is_open()) + { + log << "Can not open file " << filename; + return 0; + } + + // Write data to file + auto result = writeTreesToStream(progress, outFile, serialized_descriptors, descriptors, descriptors_count, trees, + std::move(block_getter), begin_time, end_time, pid, log); + + return result; +} + +////////////////////////////////////////////////////////////////////////// + +extern "C" PROFILER_API profiler::block_index_t writeTreesToStream(std::atomic& progress, std::ostream& str, + const profiler::SerializedData& serialized_descriptors, + const profiler::descriptors_list_t& descriptors, + profiler::block_id_t descriptors_count, + const profiler::thread_blocks_tree_t& trees, + profiler::block_getter_fn block_getter, + profiler::timestamp_t begin_time, + profiler::timestamp_t end_time, + profiler::processid_t pid, + std::ostream& log) +{ + if (trees.empty() || serialized_descriptors.empty() || descriptors_count == 0) + { + log << "Nothing to save"; + return 0; + } + + BlocksMemoryAndCount total; + + using ranges_t = std::unordered_map >; + ranges_t block_ranges; + + // Calculate block ranges and used memory (for serialization) + profiler::timestamp_t beginTime = begin_time, endTime = end_time; + size_t i = 0; + for (const auto& kv : trees) + { + const auto id = kv.first; + const auto& tree = kv.second; + + BlocksAndCSwitchesRange range; + + range.blocks = findRange(tree.children, begin_time, end_time, block_getter); + range.cswitches = findRange(tree.sync, begin_time, end_time, block_getter); + + range.blocksMemoryAndCount = calculateUsedMemoryAndBlocksCount(tree.children, range.blocks, block_getter, + descriptors, false); + total += range.blocksMemoryAndCount; + + if (range.blocksMemoryAndCount.blocksCount != 0) + { + beginTime = std::min(beginTime, block_getter(tree.children[range.blocks.begin]).node->begin()); + endTime = std::max(endTime, block_getter(tree.children[range.blocks.end - 1]).node->end()); + } + + range.cswitchesMemoryAndCount = calculateUsedMemoryAndBlocksCount(tree.sync, range.cswitches, block_getter, + descriptors, true); + total += range.cswitchesMemoryAndCount; + + if (range.cswitchesMemoryAndCount.blocksCount != 0) + { + beginTime = std::min(beginTime, block_getter(tree.children[range.cswitches.begin]).cs->begin()); + endTime = std::max(endTime, block_getter(tree.children[range.cswitches.end - 1]).cs->end()); + } + + block_ranges[id] = range; + + if (!update_progress_write(progress, 15 / static_cast(trees.size() - i), log)) + return 0; + + ++i; + } + + if (total.blocksCount == 0) + { + log << "Nothing to save"; + return 0; + } + + const uint64_t usedMemorySizeDescriptors = serialized_descriptors.size() + descriptors_count * sizeof(uint16_t); + + // Write data to stream + write(str, EASY_PROFILER_SIGNATURE); + write(str, EASY_PROFILER_VERSION); + write(str, pid); + + // write 0 because we do not need to convert time from ticks to nanoseconds (it's already converted) + write(str, 0LL); // CPU frequency + + write(str, beginTime); + write(str, endTime); + + write(str, total.usedMemorySize); + write(str, usedMemorySizeDescriptors); + write(str, total.blocksCount); + write(str, descriptors_count); + + std::vector buffer; + + // Serialize all descriptors + serializeDescriptors(str, buffer, descriptors, descriptors_count); + + // Serialize all blocks + i = 0; + for (const auto& kv : trees) + { + const auto id = kv.first; + const auto& tree = kv.second; + const auto& range = block_ranges.at(id); + + const auto nameSize = static_cast(tree.thread_name.size() + 1); + write(str, id); + write(str, nameSize); + write(str, tree.name(), nameSize); + + // Serialize context switches + write(str, range.cswitchesMemoryAndCount.blocksCount); + if (range.cswitchesMemoryAndCount.blocksCount != 0) + serializeContextSwitches(str, buffer, tree.sync, range.cswitches, block_getter); + + // Serialize blocks + write(str, range.blocksMemoryAndCount.blocksCount); + if (range.blocksMemoryAndCount.blocksCount != 0) + serializeBlocks(str, buffer, tree.children, range.blocks, block_getter, descriptors); + + if (!update_progress_write(progress, 40 + 60 / static_cast(trees.size() - i), log)) + return 0; + } + + return total.blocksCount; +} + +////////////////////////////////////////////////////////////////////////// diff --git a/profiler_gui/main_window.cpp b/profiler_gui/main_window.cpp index c73b65c..a061396 100644 --- a/profiler_gui/main_window.cpp +++ b/profiler_gui/main_window.cpp @@ -106,6 +106,7 @@ #include #include +#include #ifdef max #undef max