From b411976ca5f4291571c0cc446ee79c009b7a247c Mon Sep 17 00:00:00 2001 From: Tao Bai Date: Wed, 18 Dec 2019 11:33:39 -0800 Subject: [PATCH] [log minidump] add tool to encode/decode minidump log. - This tool could compress/encode or decode/decompress the minidump log file, will be used by script to symbolize the crash. - Added FileOutputStream and FileEncoder. Bug: crashpad:308 Change-Id: I15c3e4908882a09983ec81a90e38249967c29fc4 Reviewed-on: https://chromium-review.googlesource.com/c/crashpad/crashpad/+/1968059 Commit-Queue: Tao Bai Reviewed-by: Joshua Peraza --- tools/BUILD.gn | 13 +++ tools/base94_encoder.cc | 128 ++++++++++++++++++++++++++++++ util/BUILD.gn | 5 ++ util/stream/file_encoder.cc | 85 ++++++++++++++++++++ util/stream/file_encoder.h | 59 ++++++++++++++ util/stream/file_encoder_test.cc | 112 ++++++++++++++++++++++++++ util/stream/file_output_stream.cc | 45 +++++++++++ util/stream/file_output_stream.h | 46 +++++++++++ 8 files changed, 493 insertions(+) create mode 100644 tools/base94_encoder.cc create mode 100644 util/stream/file_encoder.cc create mode 100644 util/stream/file_encoder.h create mode 100644 util/stream/file_encoder_test.cc create mode 100644 util/stream/file_output_stream.cc create mode 100644 util/stream/file_output_stream.h diff --git a/tools/BUILD.gn b/tools/BUILD.gn index ffcd08f3..99332389 100644 --- a/tools/BUILD.gn +++ b/tools/BUILD.gn @@ -56,6 +56,19 @@ crashpad_executable("crashpad_http_upload") { ] } +crashpad_executable("base94_encoder") { + sources = [ + "base94_encoder.cc", + ] + deps = [ + ":tool_support", + "../build:default_exe_manifest_win", + "../third_party/mini_chromium:base", + "../third_party/zlib", + "../util", + ] +} + if (!crashpad_is_fuchsia) { crashpad_executable("generate_dump") { sources = [ diff --git a/tools/base94_encoder.cc b/tools/base94_encoder.cc new file mode 100644 index 00000000..d4789250 --- /dev/null +++ b/tools/base94_encoder.cc @@ -0,0 +1,128 @@ +// Copyright 2019 The Crashpad Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "base/files/file_path.h" +#include "build/build_config.h" +#include "tools/tool_support.h" +#include "util/stream/file_encoder.h" + +namespace crashpad { +namespace { + +void Usage(const base::FilePath& me) { + fprintf(stderr, +"Usage: %" PRFilePath " [options] \n" +"Encode/Decode the given file\n" +"\n" +" -e, --encode compress and encode the input file to a base94 encoded" + " file\n" +" -d, --decode decode and decompress a base94 encoded file\n" +" --help display this help and exit\n" +" --version output version information and exit\n", + me.value().c_str()); + ToolSupport::UsageTail(me); +} + +int Base94EncoderMain(int argc, char* argv[]) { + const base::FilePath argv0( + ToolSupport::CommandLineArgumentToFilePathStringType(argv[0])); + const base::FilePath me(argv0.BaseName()); + + enum OptionFlags { + // “Short” (single-character) options. + kOptionEncode = 'e', + kOptionDecode = 'd', + + // Standard options. + kOptionHelp = -2, + kOptionVersion = -3, + }; + + struct Options { + bool encoding; + base::FilePath input_file; + base::FilePath output_file; + } options = {}; + + static constexpr option long_options[] = { + {"encode", no_argument, nullptr, kOptionEncode}, + {"decode", no_argument, nullptr, kOptionDecode}, + {"help", no_argument, nullptr, kOptionHelp}, + {"version", no_argument, nullptr, kOptionVersion}, + {nullptr, 0, nullptr, 0}, + }; + + bool encoding_valid = false; + int opt; + while ((opt = getopt_long(argc, argv, "d:e", long_options, nullptr)) != -1) { + switch (opt) { + case kOptionEncode: + options.encoding = true; + encoding_valid = true; + break; + case kOptionDecode: + options.encoding = false; + encoding_valid = true; + break; + case kOptionHelp: + Usage(me); + return EXIT_SUCCESS; + case kOptionVersion: + ToolSupport::Version(me); + return EXIT_SUCCESS; + default: + ToolSupport::UsageHint(me, nullptr); + return EXIT_FAILURE; + } + } + + if (!encoding_valid) { + ToolSupport::UsageHint(me, "Either -e or -d required"); + return EXIT_FAILURE; + } + + argc -= optind; + argv += optind; + if (argc != 2) { + ToolSupport::UsageHint(me, "Both input-file and output-file required"); + return EXIT_FAILURE; + } + + options.input_file = base::FilePath( + ToolSupport::CommandLineArgumentToFilePathStringType(argv[0])); + options.output_file = base::FilePath( + ToolSupport::CommandLineArgumentToFilePathStringType(argv[1])); + + FileEncoder encoder(options.encoding ? crashpad::FileEncoder::Mode::kEncode + : crashpad::FileEncoder::Mode::kDecode, + options.input_file, + options.output_file); + return encoder.Process() ? EXIT_SUCCESS : EXIT_FAILURE; +} + +} // namespace +} // namespace crashpad + +#if defined(OS_POSIX) +int main(int argc, char* argv[]) { + return crashpad::Base94EncoderMain(argc, argv); +} +#elif defined(OS_WIN) +int wmain(int argc, wchar_t* argv[]) { + return crashpad::ToolSupport::Wmain(argc, argv, crashpad::Base94EncoderMain); +} +#endif // OS_POSIX diff --git a/util/BUILD.gn b/util/BUILD.gn index e24b675c..11d48445 100644 --- a/util/BUILD.gn +++ b/util/BUILD.gn @@ -168,6 +168,10 @@ static_library("util") { "stdlib/thread_safe_vector.h", "stream/base94_output_stream.cc", "stream/base94_output_stream.h", + "stream/file_encoder.cc", + "stream/file_encoder.h", + "stream/file_output_stream.cc", + "stream/file_output_stream.h", "stream/log_output_stream.cc", "stream/log_output_stream.h", "stream/output_stream_interface.h", @@ -593,6 +597,7 @@ source_set("util_test") { "stdlib/strnlen_test.cc", "stdlib/thread_safe_vector_test.cc", "stream/base94_output_stream_test.cc", + "stream/file_encoder_test.cc", "stream/log_output_stream_test.cc", "stream/test_output_stream.cc", "stream/test_output_stream.h", diff --git a/util/stream/file_encoder.cc b/util/stream/file_encoder.cc new file mode 100644 index 00000000..bb2c41a3 --- /dev/null +++ b/util/stream/file_encoder.cc @@ -0,0 +1,85 @@ +// Copyright 2019 The Crashpad Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "util/stream/file_encoder.h" + +#include + +#include "base/logging.h" +#include "base/macros.h" +#include "util/file/file_io.h" +#include "util/file/file_reader.h" +#include "util/file/scoped_remove_file.h" +#include "util/stream/base94_output_stream.h" +#include "util/stream/file_output_stream.h" +#include "util/stream/output_stream_interface.h" +#include "util/stream/zlib_output_stream.h" + +namespace crashpad { + +FileEncoder::FileEncoder(Mode mode, + const base::FilePath& input_path, + const base::FilePath& output_path) + : mode_(mode), input_path_(input_path), output_path_(output_path) {} + +FileEncoder::~FileEncoder() {} + +bool FileEncoder::Process() { + ScopedRemoveFile file_remover; + ScopedFileHandle write_handle(LoggingOpenFileForWrite( + output_path_, FileWriteMode::kCreateOrFail, FilePermissions::kOwnerOnly)); + if (!write_handle.is_valid()) + return false; + + // Remove the output file on failure. + file_remover.reset(output_path_); + + std::unique_ptr output; + if (mode_ == Mode::kEncode) { + output = std::make_unique( + ZlibOutputStream::Mode::kCompress, + std::make_unique( + Base94OutputStream::Mode::kEncode, + std::make_unique(write_handle.get()))); + } else { + output = std::make_unique( + Base94OutputStream::Mode::kDecode, + std::make_unique( + ZlibOutputStream::Mode::kDecompress, + std::make_unique(write_handle.get()))); + } + + FileReader file_reader; + if (!file_reader.Open(input_path_)) + return false; + + FileOperationResult read_result; + do { + uint8_t buffer[4096]; + read_result = file_reader.Read(buffer, sizeof(buffer)); + if (read_result < 0) + return false; + + if (read_result > 0 && (!output->Write(buffer, read_result))) + return false; + } while (read_result > 0); + + if (!output->Flush()) + return false; + + ignore_result(file_remover.release()); + return true; +} + +} // namespace crashpad diff --git a/util/stream/file_encoder.h b/util/stream/file_encoder.h new file mode 100644 index 00000000..2f34b0bf --- /dev/null +++ b/util/stream/file_encoder.h @@ -0,0 +1,59 @@ +// Copyright 2019 The Crashpad Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CRASHPAD_UTIL_STREAM_FILE_ENCODER_H_ +#define CRASHPAD_UTIL_STREAM_FILE_ENCODER_H_ + +#include "base/files/file_path.h" +#include "base/macros.h" + +namespace crashpad { + +//! \brief The class is used to compress and base94-encode, or base94-decode +//! and decompress the given input file to the output file. +class FileEncoder { + public: + //! \brief Whether this object is configured to encode or decode data. + enum class Mode : bool { + //! \brief Data passed through this object is encoded. + kEncode = false, + //! \brief Data passed through this object is decoded. + kDecode = true + }; + + //! \param[in] mode The work mode of this object. + //! \param[in] input_path The input file that this object reads from. + //! \param[in] output_path The output file that this object writes to. + FileEncoder(Mode mode, + const base::FilePath& input_path, + const base::FilePath& output_path); + ~FileEncoder(); + + //! \brief Encode/decode the data from \a input_path_ file according work + //! \a mode, and write the result to \a output_path_ on success. + //! + //! \return `true` on success. + bool Process(); + + private: + Mode mode_; + base::FilePath input_path_; + base::FilePath output_path_; + + DISALLOW_COPY_AND_ASSIGN(FileEncoder); +}; + +} // namespace crashpad + +#endif // CRASHPAD_UTIL_STREAM_FILE_ENCODER_H_ diff --git a/util/stream/file_encoder_test.cc b/util/stream/file_encoder_test.cc new file mode 100644 index 00000000..ff989146 --- /dev/null +++ b/util/stream/file_encoder_test.cc @@ -0,0 +1,112 @@ +// Copyright 2019 The Crashpad Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "util/stream/file_encoder.h" + +#include +#include +#include + +#include "base/files/file_path.h" +#include "base/macros.h" +#include "gtest/gtest.h" +#include "test/scoped_temp_dir.h" +#include "util/file/file_io.h" +#include "util/stream/file_output_stream.h" + +namespace crashpad { +namespace test { +namespace { + +constexpr size_t kBufferSize = 4096; + +class FileEncoderTest : public testing::Test { + public: + FileEncoderTest() {} + + void Verify(size_t size) { + std::string contents; + ASSERT_TRUE(LoggingReadEntireFile(decoded_, &contents)); + ASSERT_EQ(contents.size(), size); + EXPECT_EQ(memcmp(deterministic_input_.get(), contents.data(), size), 0); + } + + const uint8_t* BuildDeterministicInput(size_t size) { + deterministic_input_ = std::make_unique(size); + uint8_t* deterministic_input_base = deterministic_input_.get(); + while (size-- > 0) + deterministic_input_base[size] = static_cast(size); + return deterministic_input_base; + } + + void GenerateOrigFile(size_t size) { + ScopedFileHandle write_handle(OpenFileForWrite( + orig_, FileWriteMode::kCreateOrFail, FilePermissions::kOwnerOnly)); + ASSERT_TRUE(write_handle.is_valid()); + FileOutputStream out(write_handle.get()); + const uint8_t* buf = BuildDeterministicInput(size); + while (size > 0) { + size_t m = std::min(kBufferSize, size); + ASSERT_TRUE(out.Write(buf, m)); + size -= m; + buf += m; + } + ASSERT_TRUE(out.Flush()); + } + + FileEncoder* encoder() const { return encoder_.get(); } + + FileEncoder* decoder() const { return decoder_.get(); } + + protected: + void SetUp() override { + temp_dir_ = std::make_unique(); + orig_ = base::FilePath(temp_dir_->path().Append(FILE_PATH_LITERAL("orig"))); + encoded_ = + base::FilePath(temp_dir_->path().Append(FILE_PATH_LITERAL("encoded"))); + decoded_ = + base::FilePath(temp_dir_->path().Append(FILE_PATH_LITERAL("decoded"))); + encoder_ = std::make_unique( + FileEncoder::Mode::kEncode, orig_, encoded_); + decoder_ = std::make_unique( + FileEncoder::Mode::kDecode, encoded_, decoded_); + } + + private: + std::unique_ptr temp_dir_; + base::FilePath orig_; + base::FilePath encoded_; + base::FilePath decoded_; + std::unique_ptr encoder_; + std::unique_ptr decoder_; + std::unique_ptr deterministic_input_; +}; + +TEST_F(FileEncoderTest, ProcessShortFile) { + GenerateOrigFile(kBufferSize - 512); + EXPECT_TRUE(encoder()->Process()); + EXPECT_TRUE(decoder()->Process()); + Verify(kBufferSize - 512); +} + +TEST_F(FileEncoderTest, ProcessLongFile) { + GenerateOrigFile(kBufferSize + 512); + EXPECT_TRUE(encoder()->Process()); + EXPECT_TRUE(decoder()->Process()); + Verify(kBufferSize + 512); +} + +} // namespace +} // namespace test +} // namespace crashpad diff --git a/util/stream/file_output_stream.cc b/util/stream/file_output_stream.cc new file mode 100644 index 00000000..022d91fe --- /dev/null +++ b/util/stream/file_output_stream.cc @@ -0,0 +1,45 @@ +// Copyright 2019 The Crashpad Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "util/stream/file_output_stream.h" + +#include "base/logging.h" + +namespace crashpad { + +FileOutputStream::FileOutputStream(FileHandle file_handle) + : writer_(file_handle), flush_needed_(false), flushed_(false) {} + +FileOutputStream::~FileOutputStream() { + DCHECK(!flush_needed_); +} + +bool FileOutputStream::Write(const uint8_t* data, size_t size) { + DCHECK(!flushed_); + + if (!writer_.Write(data, size)) { + LOG(ERROR) << "Write: Failed"; + return false; + } + flush_needed_ = true; + return true; +} + +bool FileOutputStream::Flush() { + flush_needed_ = false; + flushed_ = true; + return true; +} + +} // namespace crashpad diff --git a/util/stream/file_output_stream.h b/util/stream/file_output_stream.h new file mode 100644 index 00000000..128ccd5e --- /dev/null +++ b/util/stream/file_output_stream.h @@ -0,0 +1,46 @@ +// Copyright 2019 The Crashpad Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CRASHPAD_UTIL_STREAM_FILE_OUTPUT_STREAM_H_ +#define CRASHPAD_UTIL_STREAM_FILE_OUTPUT_STREAM_H_ + +#include "base/macros.h" +#include "util/file/file_io.h" +#include "util/file/file_writer.h" +#include "util/stream/output_stream_interface.h" + +namespace crashpad { + +//! \brief The class is used to write data to a file. +class FileOutputStream : public OutputStreamInterface { + public: + //! \param[in] file_handle The file that this object writes to. + explicit FileOutputStream(FileHandle file_handle); + ~FileOutputStream(); + + // OutputStream. + bool Write(const uint8_t* data, size_t size) override; + bool Flush() override; + + private: + WeakFileHandleFileWriter writer_; + bool flush_needed_; + bool flushed_; + + DISALLOW_COPY_AND_ASSIGN(FileOutputStream); +}; + +} // namespace crashpad + +#endif // CRASHPAD_UTIL_STREAM_FILE_OUTPUT_STREAM_H_