[log minidump] add tool to encode/decode minidump log.

- This tool could compress/encode or decode/decompress the minidump
  log file, will be used by script to symbolize the crash.
- Added FileOutputStream and FileEncoder.

Bug: crashpad:308
Change-Id: I15c3e4908882a09983ec81a90e38249967c29fc4
Reviewed-on: https://chromium-review.googlesource.com/c/crashpad/crashpad/+/1968059
Commit-Queue: Tao Bai <michaelbai@chromium.org>
Reviewed-by: Joshua Peraza <jperaza@chromium.org>

tools/BUILD.gn

@@ -56,6 +56,19 @@ crashpad_executable("crashpad_http_upload") {
   ]
 }
 
+crashpad_executable("base94_encoder") {
+  sources = [
+    "base94_encoder.cc",
+  ]
+  deps = [
+    ":tool_support",
+    "../build:default_exe_manifest_win",
+    "../third_party/mini_chromium:base",
+    "../third_party/zlib",
+    "../util",
+  ]
+}
+
 if (!crashpad_is_fuchsia) {
   crashpad_executable("generate_dump") {
     sources = [

tools/base94_encoder.cc

@@ -0,0 +1,128 @@
+// Copyright 2019 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <getopt.h>
+#include <stdio.h>
+
+#include "base/files/file_path.h"
+#include "build/build_config.h"
+#include "tools/tool_support.h"
+#include "util/stream/file_encoder.h"
+
+namespace crashpad {
+namespace {
+
+void Usage(const base::FilePath& me) {
+  fprintf(stderr,
+"Usage: %" PRFilePath " [options] <input-file> <output-file>\n"
+"Encode/Decode the given file\n"
+"\n"
+"  -e, --encode   compress and encode the input file to a base94 encoded"
+                  " file\n"
+"  -d, --decode   decode and decompress a base94 encoded file\n"
+"      --help     display this help and exit\n"
+"      --version  output version information and exit\n",
+          me.value().c_str());
+  ToolSupport::UsageTail(me);
+}
+
+int Base94EncoderMain(int argc, char* argv[]) {
+  const base::FilePath argv0(
+      ToolSupport::CommandLineArgumentToFilePathStringType(argv[0]));
+  const base::FilePath me(argv0.BaseName());
+
+  enum OptionFlags {
+    // “Short” (single-character) options.
+    kOptionEncode = 'e',
+    kOptionDecode = 'd',
+
+    // Standard options.
+    kOptionHelp = -2,
+    kOptionVersion = -3,
+  };
+
+  struct Options {
+    bool encoding;
+    base::FilePath input_file;
+    base::FilePath output_file;
+  } options = {};
+
+  static constexpr option long_options[] = {
+      {"encode", no_argument, nullptr, kOptionEncode},
+      {"decode", no_argument, nullptr, kOptionDecode},
+      {"help", no_argument, nullptr, kOptionHelp},
+      {"version", no_argument, nullptr, kOptionVersion},
+      {nullptr, 0, nullptr, 0},
+  };
+
+  bool encoding_valid = false;
+  int opt;
+  while ((opt = getopt_long(argc, argv, "d:e", long_options, nullptr)) != -1) {
+    switch (opt) {
+      case kOptionEncode:
+        options.encoding = true;
+        encoding_valid = true;
+        break;
+      case kOptionDecode:
+        options.encoding = false;
+        encoding_valid = true;
+        break;
+      case kOptionHelp:
+        Usage(me);
+        return EXIT_SUCCESS;
+      case kOptionVersion:
+        ToolSupport::Version(me);
+        return EXIT_SUCCESS;
+      default:
+        ToolSupport::UsageHint(me, nullptr);
+        return EXIT_FAILURE;
+    }
+  }
+
+  if (!encoding_valid) {
+    ToolSupport::UsageHint(me, "Either -e or -d required");
+    return EXIT_FAILURE;
+  }
+
+  argc -= optind;
+  argv += optind;
+  if (argc != 2) {
+    ToolSupport::UsageHint(me, "Both input-file and output-file required");
+    return EXIT_FAILURE;
+  }
+
+  options.input_file = base::FilePath(
+      ToolSupport::CommandLineArgumentToFilePathStringType(argv[0]));
+  options.output_file = base::FilePath(
+      ToolSupport::CommandLineArgumentToFilePathStringType(argv[1]));
+
+  FileEncoder encoder(options.encoding ? crashpad::FileEncoder::Mode::kEncode
+                                       : crashpad::FileEncoder::Mode::kDecode,
+                      options.input_file,
+                      options.output_file);
+  return encoder.Process() ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+}  // namespace
+}  // namespace crashpad
+
+#if defined(OS_POSIX)
+int main(int argc, char* argv[]) {
+  return crashpad::Base94EncoderMain(argc, argv);
+}
+#elif defined(OS_WIN)
+int wmain(int argc, wchar_t* argv[]) {
+  return crashpad::ToolSupport::Wmain(argc, argv, crashpad::Base94EncoderMain);
+}
+#endif  // OS_POSIX

util/BUILD.gn

@@ -168,6 +168,10 @@ static_library("util") {
     "stdlib/thread_safe_vector.h",
     "stream/base94_output_stream.cc",
     "stream/base94_output_stream.h",
+    "stream/file_encoder.cc",
+    "stream/file_encoder.h",
+    "stream/file_output_stream.cc",
+    "stream/file_output_stream.h",
     "stream/log_output_stream.cc",
     "stream/log_output_stream.h",
     "stream/output_stream_interface.h",
@@ -593,6 +597,7 @@ source_set("util_test") {
     "stdlib/strnlen_test.cc",
     "stdlib/thread_safe_vector_test.cc",
     "stream/base94_output_stream_test.cc",
+    "stream/file_encoder_test.cc",
     "stream/log_output_stream_test.cc",
     "stream/test_output_stream.cc",
     "stream/test_output_stream.h",

util/stream/file_encoder.cc

@@ -0,0 +1,85 @@
+// Copyright 2019 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "util/stream/file_encoder.h"
+
+#include <memory>
+
+#include "base/logging.h"
+#include "base/macros.h"
+#include "util/file/file_io.h"
+#include "util/file/file_reader.h"
+#include "util/file/scoped_remove_file.h"
+#include "util/stream/base94_output_stream.h"
+#include "util/stream/file_output_stream.h"
+#include "util/stream/output_stream_interface.h"
+#include "util/stream/zlib_output_stream.h"
+
+namespace crashpad {
+
+FileEncoder::FileEncoder(Mode mode,
+                         const base::FilePath& input_path,
+                         const base::FilePath& output_path)
+    : mode_(mode), input_path_(input_path), output_path_(output_path) {}
+
+FileEncoder::~FileEncoder() {}
+
+bool FileEncoder::Process() {
+  ScopedRemoveFile file_remover;
+  ScopedFileHandle write_handle(LoggingOpenFileForWrite(
+      output_path_, FileWriteMode::kCreateOrFail, FilePermissions::kOwnerOnly));
+  if (!write_handle.is_valid())
+    return false;
+
+  // Remove the output file on failure.
+  file_remover.reset(output_path_);
+
+  std::unique_ptr<OutputStreamInterface> output;
+  if (mode_ == Mode::kEncode) {
+    output = std::make_unique<ZlibOutputStream>(
+        ZlibOutputStream::Mode::kCompress,
+        std::make_unique<Base94OutputStream>(
+            Base94OutputStream::Mode::kEncode,
+            std::make_unique<FileOutputStream>(write_handle.get())));
+  } else {
+    output = std::make_unique<Base94OutputStream>(
+        Base94OutputStream::Mode::kDecode,
+        std::make_unique<ZlibOutputStream>(
+            ZlibOutputStream::Mode::kDecompress,
+            std::make_unique<FileOutputStream>(write_handle.get())));
+  }
+
+  FileReader file_reader;
+  if (!file_reader.Open(input_path_))
+    return false;
+
+  FileOperationResult read_result;
+  do {
+    uint8_t buffer[4096];
+    read_result = file_reader.Read(buffer, sizeof(buffer));
+    if (read_result < 0)
+      return false;
+
+    if (read_result > 0 && (!output->Write(buffer, read_result)))
+      return false;
+  } while (read_result > 0);
+
+  if (!output->Flush())
+    return false;
+
+  ignore_result(file_remover.release());
+  return true;
+}
+
+}  // namespace crashpad

util/stream/file_encoder.h

@@ -0,0 +1,59 @@
+// Copyright 2019 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef CRASHPAD_UTIL_STREAM_FILE_ENCODER_H_
+#define CRASHPAD_UTIL_STREAM_FILE_ENCODER_H_
+
+#include "base/files/file_path.h"
+#include "base/macros.h"
+
+namespace crashpad {
+
+//! \brief The class is used to compress and base94-encode, or base94-decode
+//! and decompress the given input file to the output file.
+class FileEncoder {
+ public:
+  //! \brief Whether this object is configured to encode or decode data.
+  enum class Mode : bool {
+    //! \brief Data passed through this object is encoded.
+    kEncode = false,
+    //! \brief Data passed through this object is decoded.
+    kDecode = true
+  };
+
+  //! \param[in] mode The work mode of this object.
+  //! \param[in] input_path The input file that this object reads from.
+  //! \param[in] output_path The output file that this object writes to.
+  FileEncoder(Mode mode,
+              const base::FilePath& input_path,
+              const base::FilePath& output_path);
+  ~FileEncoder();
+
+  //! \brief Encode/decode the data from \a input_path_ file according work
+  //! \a mode, and write the result to \a output_path_ on success.
+  //!
+  //! \return `true` on success.
+  bool Process();
+
+ private:
+  Mode mode_;
+  base::FilePath input_path_;
+  base::FilePath output_path_;
+
+  DISALLOW_COPY_AND_ASSIGN(FileEncoder);
+};
+
+}  // namespace crashpad
+
+#endif  // CRASHPAD_UTIL_STREAM_FILE_ENCODER_H_

util/stream/file_encoder_test.cc

@@ -0,0 +1,112 @@
+// Copyright 2019 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "util/stream/file_encoder.h"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "base/files/file_path.h"
+#include "base/macros.h"
+#include "gtest/gtest.h"
+#include "test/scoped_temp_dir.h"
+#include "util/file/file_io.h"
+#include "util/stream/file_output_stream.h"
+
+namespace crashpad {
+namespace test {
+namespace {
+
+constexpr size_t kBufferSize = 4096;
+
+class FileEncoderTest : public testing::Test {
+ public:
+  FileEncoderTest() {}
+
+  void Verify(size_t size) {
+    std::string contents;
+    ASSERT_TRUE(LoggingReadEntireFile(decoded_, &contents));
+    ASSERT_EQ(contents.size(), size);
+    EXPECT_EQ(memcmp(deterministic_input_.get(), contents.data(), size), 0);
+  }
+
+  const uint8_t* BuildDeterministicInput(size_t size) {
+    deterministic_input_ = std::make_unique<uint8_t[]>(size);
+    uint8_t* deterministic_input_base = deterministic_input_.get();
+    while (size-- > 0)
+      deterministic_input_base[size] = static_cast<uint8_t>(size);
+    return deterministic_input_base;
+  }
+
+  void GenerateOrigFile(size_t size) {
+    ScopedFileHandle write_handle(OpenFileForWrite(
+        orig_, FileWriteMode::kCreateOrFail, FilePermissions::kOwnerOnly));
+    ASSERT_TRUE(write_handle.is_valid());
+    FileOutputStream out(write_handle.get());
+    const uint8_t* buf = BuildDeterministicInput(size);
+    while (size > 0) {
+      size_t m = std::min(kBufferSize, size);
+      ASSERT_TRUE(out.Write(buf, m));
+      size -= m;
+      buf += m;
+    }
+    ASSERT_TRUE(out.Flush());
+  }
+
+  FileEncoder* encoder() const { return encoder_.get(); }
+
+  FileEncoder* decoder() const { return decoder_.get(); }
+
+ protected:
+  void SetUp() override {
+    temp_dir_ = std::make_unique<ScopedTempDir>();
+    orig_ = base::FilePath(temp_dir_->path().Append(FILE_PATH_LITERAL("orig")));
+    encoded_ =
+        base::FilePath(temp_dir_->path().Append(FILE_PATH_LITERAL("encoded")));
+    decoded_ =
+        base::FilePath(temp_dir_->path().Append(FILE_PATH_LITERAL("decoded")));
+    encoder_ = std::make_unique<FileEncoder>(
+        FileEncoder::Mode::kEncode, orig_, encoded_);
+    decoder_ = std::make_unique<FileEncoder>(
+        FileEncoder::Mode::kDecode, encoded_, decoded_);
+  }
+
+ private:
+  std::unique_ptr<ScopedTempDir> temp_dir_;
+  base::FilePath orig_;
+  base::FilePath encoded_;
+  base::FilePath decoded_;
+  std::unique_ptr<FileEncoder> encoder_;
+  std::unique_ptr<FileEncoder> decoder_;
+  std::unique_ptr<uint8_t[]> deterministic_input_;
+};
+
+TEST_F(FileEncoderTest, ProcessShortFile) {
+  GenerateOrigFile(kBufferSize - 512);
+  EXPECT_TRUE(encoder()->Process());
+  EXPECT_TRUE(decoder()->Process());
+  Verify(kBufferSize - 512);
+}
+
+TEST_F(FileEncoderTest, ProcessLongFile) {
+  GenerateOrigFile(kBufferSize + 512);
+  EXPECT_TRUE(encoder()->Process());
+  EXPECT_TRUE(decoder()->Process());
+  Verify(kBufferSize + 512);
+}
+
+}  // namespace
+}  // namespace test
+}  // namespace crashpad

util/stream/file_output_stream.cc

@@ -0,0 +1,45 @@
+// Copyright 2019 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "util/stream/file_output_stream.h"
+
+#include "base/logging.h"
+
+namespace crashpad {
+
+FileOutputStream::FileOutputStream(FileHandle file_handle)
+    : writer_(file_handle), flush_needed_(false), flushed_(false) {}
+
+FileOutputStream::~FileOutputStream() {
+  DCHECK(!flush_needed_);
+}
+
+bool FileOutputStream::Write(const uint8_t* data, size_t size) {
+  DCHECK(!flushed_);
+
+  if (!writer_.Write(data, size)) {
+    LOG(ERROR) << "Write: Failed";
+    return false;
+  }
+  flush_needed_ = true;
+  return true;
+}
+
+bool FileOutputStream::Flush() {
+  flush_needed_ = false;
+  flushed_ = true;
+  return true;
+}
+
+}  // namespace crashpad

util/stream/file_output_stream.h

@@ -0,0 +1,46 @@
+// Copyright 2019 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef CRASHPAD_UTIL_STREAM_FILE_OUTPUT_STREAM_H_
+#define CRASHPAD_UTIL_STREAM_FILE_OUTPUT_STREAM_H_
+
+#include "base/macros.h"
+#include "util/file/file_io.h"
+#include "util/file/file_writer.h"
+#include "util/stream/output_stream_interface.h"
+
+namespace crashpad {
+
+//! \brief The class is used to write data to a file.
+class FileOutputStream : public OutputStreamInterface {
+ public:
+  //! \param[in] file_handle The file that this object writes to.
+  explicit FileOutputStream(FileHandle file_handle);
+  ~FileOutputStream();
+
+  // OutputStream.
+  bool Write(const uint8_t* data, size_t size) override;
+  bool Flush() override;
+
+ private:
+  WeakFileHandleFileWriter writer_;
+  bool flush_needed_;
+  bool flushed_;
+
+  DISALLOW_COPY_AND_ASSIGN(FileOutputStream);
+};
+
+}  // namespace crashpad
+
+#endif  // CRASHPAD_UTIL_STREAM_FILE_OUTPUT_STREAM_H_