Add a cross-platform database implementation

This CL, based on
https://chromium-review.googlesource.com/c/crashpad/crashpad/+/689745
adds a cross-platform database implementation side-by-side with the
existing macOS and Windows implementations. The generic implementation
is used for Linux, Android and Fuchsia.

The database uses the directory structure from the macOS
implementation, but stores report metadata in companion files for each
report, rather than using filesystem attributes. The database uses
lockfiles (companion files opened with O_EXCL) to protect report access
because they are widely supported across filesystems. Lost lockfiles
are removed after 3 days, along with any reports or metadata they were
protecting.

Bug: crashpad:206
Change-Id: I086e9001350e4446dd2f8c12fd3817377f509d3e
Reviewed-on: https://chromium-review.googlesource.com/919527
Commit-Queue: Joshua Peraza <jperaza@chromium.org>
Reviewed-by: Mark Mentovai <mark@chromium.org>

client/BUILD.gn

@@ -61,11 +61,14 @@ static_library("client") {
 
   if (crashpad_is_fuchsia) {
     sources += [
-      "crash_report_database_fuchsia.cc",
       "crashpad_client_fuchsia.cc",
     ]
   }
 
+  if (crashpad_is_linux || crashpad_is_android || crashpad_is_fuchsia) {
+    sources += [ "crash_report_database_generic.cc" ]
+  }
+
   public_configs = [ "..:crashpad_config" ]
 
   deps = [

client/client.gyp

@@ -35,6 +35,7 @@
         'annotation_list.h',
         'crash_report_database.cc',
         'crash_report_database.h',
+        'crash_report_database_generic.cc',
         'crash_report_database_mac.mm',
         'crash_report_database_win.cc',
         'crashpad_client.h',

client/client_test.gyp

@@ -51,6 +51,9 @@
             '../handler/handler.gyp:crashpad_handler_console',
           ],
         }],
+        ['OS=="linux" or OS=="android"',
+          {'dependencies!': ['../handler/handler.gyp:crashpad_handler']},
+        ],
       ],
     },
   ],

client/crash_report_database.h

@@ -115,7 +115,7 @@ class CrashReportDatabase {
     const UUID& ReportID() { return uuid_; }
 
    private:
-    friend class CrashReportDatabase;
+    friend class CrashReportDatabaseGeneric;
     friend class CrashReportDatabaseMac;
     friend class CrashReportDatabaseWin;
 
@@ -142,6 +142,7 @@ class CrashReportDatabase {
 
    private:
     friend class CrashReportDatabase;
+    friend class CrashReportDatabaseGeneric;
     friend class CrashReportDatabaseMac;
     friend class CrashReportDatabaseWin;
 
@@ -348,6 +349,17 @@ class CrashReportDatabase {
   //! \return The operation status code.
   virtual OperationStatus RequestUpload(const UUID& uuid) = 0;
 
+  //! \brief Cleans the database of expired lockfiles, metadata without report
+  //!     files, and report files without metadata.
+  //!
+  //! This method does nothing on the macOS and Windows implementations of the
+  //! database.
+  //!
+  //! \param[in] lockfile_ttl The number of seconds at which lockfiles or new
+  //!     report files are considered expired.
+  //! \return The number of reports cleaned.
+  virtual int CleanDatabase(time_t lockfile_ttl) { return 0; }
+
  protected:
   CrashReportDatabase() {}
 

client/crash_report_database_fuchsia.cc

@@ -1,35 +0,0 @@
-// Copyright 2017 The Crashpad Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "client/crash_report_database.h"
-
-#include "base/logging.h"
-
-namespace crashpad {
-
-// static
-std::unique_ptr<CrashReportDatabase> CrashReportDatabase::Initialize(
-    const base::FilePath& path) {
-  NOTREACHED();  // TODO(scottmg): https://crashpad.chromium.org/bug/196
-  return std::unique_ptr<CrashReportDatabase>();
-}
-
-// static
-std::unique_ptr<CrashReportDatabase>
-CrashReportDatabase::InitializeWithoutCreating(const base::FilePath& path) {
-  NOTREACHED();  // TODO(scottmg): https://crashpad.chromium.org/bug/196
-  return std::unique_ptr<CrashReportDatabase>();
-}
-
-}  // namespace crashpad

client/crash_report_database_generic.cc

@@ -0,0 +1,841 @@
+// Copyright 2018 The Crashpad Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "client/crash_report_database.h"
+
+#include <stdint.h>
+#include <sys/types.h>
+
+#include <utility>
+
+#include "base/logging.h"
+#include "build/build_config.h"
+#include "client/settings.h"
+#include "util/file/directory_reader.h"
+#include "util/file/filesystem.h"
+#include "util/misc/initialization_state_dcheck.h"
+
+namespace crashpad {
+
+namespace {
+
+// Reads from the current file position to EOF and returns as a string of bytes.
+bool ReadRestOfFileAsString(FileHandle handle, std::string* contents) {
+  char buffer[4096];
+  FileOperationResult rv;
+  std::string local_contents;
+  while ((rv = ReadFile(handle, buffer, sizeof(buffer))) > 0) {
+    local_contents.append(buffer, rv);
+  }
+  if (rv < 0) {
+    PLOG(ERROR) << "ReadFile";
+    return false;
+  }
+  contents->swap(local_contents);
+  return true;
+}
+
+base::FilePath ReplaceFinalExtension(
+    const base::FilePath& path,
+    const base::FilePath::StringType extension) {
+  return base::FilePath(path.RemoveFinalExtension().value() + extension);
+}
+
+using OperationStatus = CrashReportDatabase::OperationStatus;
+
+constexpr base::FilePath::CharType kSettings[] =
+    FILE_PATH_LITERAL("settings.dat");
+
+constexpr base::FilePath::CharType kCrashReportExtension[] =
+    FILE_PATH_LITERAL(".dmp");
+constexpr base::FilePath::CharType kMetadataExtension[] =
+    FILE_PATH_LITERAL(".meta");
+constexpr base::FilePath::CharType kLockExtension[] =
+    FILE_PATH_LITERAL(".lock");
+
+constexpr base::FilePath::CharType kNewDirectory[] = FILE_PATH_LITERAL("new");
+constexpr base::FilePath::CharType kPendingDirectory[] =
+    FILE_PATH_LITERAL("pending");
+constexpr base::FilePath::CharType kCompletedDirectory[] =
+    FILE_PATH_LITERAL("completed");
+
+constexpr const base::FilePath::CharType* kReportDirectories[] = {
+    kNewDirectory,
+    kPendingDirectory,
+    kCompletedDirectory,
+};
+
+enum {
+  //! \brief Corresponds to uploaded bit of the report state.
+  kAttributeUploaded = 1 << 0,
+
+  //! \brief Corresponds to upload_explicity_requested bit of the report state.
+  kAttributeUploadExplicitlyRequested = 1 << 1,
+};
+
+struct ReportMetadata {
+  static constexpr int32_t kVersion = 1;
+
+  int32_t version = kVersion;
+  int32_t upload_attempts = 0;
+  int64_t last_upload_attempt_time = 0;
+  time_t creation_time = 0;
+  uint8_t attributes = 0;
+};
+
+// A lock held while using database resources.
+class ScopedLockFile {
+ public:
+  ScopedLockFile() = default;
+  ~ScopedLockFile() = default;
+
+  ScopedLockFile& operator=(ScopedLockFile&& other) {
+    lock_file_.reset(other.lock_file_.release());
+    return *this;
+  }
+
+  // Attempt to acquire a lock for the report at report_path.
+  // Return `true` on success, otherwise `false`.
+  bool ResetAcquire(const base::FilePath& report_path) {
+    lock_file_.reset();
+
+    base::FilePath lock_path(report_path.RemoveFinalExtension().value() +
+                             kLockExtension);
+    ScopedFileHandle lock_fd(LoggingOpenFileForWrite(
+        lock_path, FileWriteMode::kCreateOrFail, FilePermissions::kOwnerOnly));
+    if (!lock_fd.is_valid()) {
+      return false;
+    }
+    lock_file_.reset(lock_path);
+
+    time_t timestamp = time(nullptr);
+    if (!LoggingWriteFile(lock_fd.get(), &timestamp, sizeof(timestamp))) {
+      return false;
+    }
+
+    return true;
+  }
+
+  // Returns `true` if the lock is held.
+  bool is_valid() const { return lock_file_.is_valid(); }
+
+  // Returns `true` if the lockfile at lock_path has expired.
+  static bool IsExpired(const base::FilePath& lock_path, time_t lockfile_ttl) {
+    time_t now = time(nullptr);
+
+    timespec filetime;
+    if (FileModificationTime(lock_path, &filetime) &&
+        filetime.tv_sec > now + lockfile_ttl) {
+      return false;
+    }
+
+    ScopedFileHandle lock_fd(LoggingOpenFileForReadAndWrite(
+        lock_path, FileWriteMode::kReuseOrFail, FilePermissions::kOwnerOnly));
+    if (!lock_fd.is_valid()) {
+      return false;
+    }
+
+    time_t timestamp;
+    if (!LoggingReadFileExactly(lock_fd.get(), &timestamp, sizeof(timestamp))) {
+      return false;
+    }
+
+    return now >= timestamp + lockfile_ttl;
+  }
+
+ private:
+  ScopedRemoveFile lock_file_;
+};
+
+}  // namespace
+
+class CrashReportDatabaseGeneric : public CrashReportDatabase {
+ public:
+  CrashReportDatabaseGeneric();
+  ~CrashReportDatabaseGeneric() override;
+
+  bool Initialize(const base::FilePath& path, bool may_create);
+
+  // CrashReportDatabase:
+  Settings* GetSettings() override;
+  OperationStatus PrepareNewCrashReport(
+      std::unique_ptr<NewReport>* report) override;
+  OperationStatus FinishedWritingCrashReport(std::unique_ptr<NewReport> report,
+                                             UUID* uuid) override;
+  OperationStatus LookUpCrashReport(const UUID& uuid, Report* report) override;
+  OperationStatus GetPendingReports(std::vector<Report>* reports) override;
+  OperationStatus GetCompletedReports(std::vector<Report>* reports) override;
+  OperationStatus GetReportForUploading(
+      const UUID& uuid,
+      std::unique_ptr<const UploadReport>* report) override;
+  OperationStatus SkipReportUpload(const UUID& uuid,
+                                   Metrics::CrashSkippedReason reason) override;
+  OperationStatus DeleteReport(const UUID& uuid) override;
+  OperationStatus RequestUpload(const UUID& uuid) override;
+  int CleanDatabase(time_t lockfile_ttl) override;
+
+ private:
+  struct LockfileUploadReport : public UploadReport {
+    ScopedLockFile lock_file;
+  };
+
+  enum ReportState : int32_t {
+    kUninitialized = -1,
+
+    // Being created by a caller of PrepareNewCrashReport().
+    kNew,
+
+    // Created by FinishedWritingCrashReport(), but not yet uploaded.
+    kPending,
+
+    // Upload completed or skipped.
+    kCompleted,
+
+    // Specifies either kPending or kCompleted.
+    kSearchable,
+  };
+
+  // CrashReportDatabase:
+  OperationStatus RecordUploadAttempt(UploadReport* report,
+                                      bool successful,
+                                      const std::string& id) override;
+
+  // Builds a filepath for the report with the specified uuid and state.
+  base::FilePath ReportPath(const UUID& uuid, ReportState state);
+
+  // Locates the report with id uuid and returns its file path in path and a
+  // lock for the report in lock_file. This method succeeds as long as the
+  // report file exists and the lock can be acquired. No validation is done on
+  // the existence or content of the metadata file.
+  OperationStatus LocateAndLockReport(const UUID& uuid,
+                                      ReportState state,
+                                      base::FilePath* path,
+                                      ScopedLockFile* lock_file);
+
+  // Locates, locks, and reads the metadata for the report with the specified
+  // uuid and state. This method will fail and may remove reports if invalid
+  // metadata is detected. state may be kPending, kCompleted, or kSearchable.
+  OperationStatus CheckoutReport(const UUID& uuid,
+                                 ReportState state,
+                                 base::FilePath* path,
+                                 ScopedLockFile* lock_file,
+                                 Report* report);
+
+  // Reads metadata for all reports in state and returns it in reports.
+  OperationStatus ReportsInState(ReportState state,
+                                 std::vector<Report>* reports);
+
+  // Cleans lone metadata, reports, or expired locks in a particular state.
+  int CleanReportsInState(ReportState state, time_t lockfile_ttl);
+
+  // Reads the metadata for a report from path and returns it in report.
+  static bool ReadMetadata(const base::FilePath& path, Report* report);
+
+  // Wraps ReadMetadata and removes the report from the database on failure.
+  static bool CleaningReadMetadata(const base::FilePath& path, Report* report);
+
+  // Writes metadata for a new report to the filesystem at path.
+  static bool WriteNewMetadata(const base::FilePath& path);
+
+  // Writes the metadata for report to the filesystem at path.
+  static bool WriteMetadata(const base::FilePath& path, const Report& report);
+
+  base::FilePath base_dir_;
+  Settings settings_;
+  InitializationStateDcheck initialized_;
+
+  DISALLOW_COPY_AND_ASSIGN(CrashReportDatabaseGeneric);
+};
+
+CrashReportDatabaseGeneric::CrashReportDatabaseGeneric() = default;
+
+CrashReportDatabaseGeneric::~CrashReportDatabaseGeneric() = default;
+
+bool CrashReportDatabaseGeneric::Initialize(const base::FilePath& path,
+                                            bool may_create) {
+  INITIALIZATION_STATE_SET_INITIALIZING(initialized_);
+  base_dir_ = path;
+
+  if (!IsDirectory(base_dir_, true) &&
+      !(may_create &&
+        LoggingCreateDirectory(base_dir_, FilePermissions::kOwnerOnly, true))) {
+    return false;
+  }
+
+  for (size_t i = 0; i < arraysize(kReportDirectories); ++i) {
+    if (!LoggingCreateDirectory(base_dir_.Append(kReportDirectories[i]),
+                                FilePermissions::kOwnerOnly,
+                                true)) {
+      return false;
+    }
+  }
+
+  if (!settings_.Initialize(base_dir_.Append(kSettings))) {
+    return false;
+  }
+
+  INITIALIZATION_STATE_SET_VALID(initialized_);
+  return true;
+}
+
+// static
+std::unique_ptr<CrashReportDatabase> CrashReportDatabase::Initialize(
+    const base::FilePath& path) {
+  auto database = std::make_unique<CrashReportDatabaseGeneric>();
+  return database->Initialize(path, true) ? std::move(database) : nullptr;
+}
+
+// static
+std::unique_ptr<CrashReportDatabase>
+CrashReportDatabase::InitializeWithoutCreating(const base::FilePath& path) {
+  auto database = std::make_unique<CrashReportDatabaseGeneric>();
+  return database->Initialize(path, false) ? std::move(database) : nullptr;
+}
+
+Settings* CrashReportDatabaseGeneric::GetSettings() {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+  return &settings_;
+}
+
+OperationStatus CrashReportDatabaseGeneric::PrepareNewCrashReport(
+    std::unique_ptr<NewReport>* report) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  auto new_report = std::make_unique<NewReport>();
+  if (!new_report->Initialize(base_dir_.Append(kNewDirectory),
+                              kCrashReportExtension)) {
+    return kFileSystemError;
+  }
+
+  report->reset(new_report.release());
+  return kNoError;
+}
+
+OperationStatus CrashReportDatabaseGeneric::FinishedWritingCrashReport(
+    std::unique_ptr<NewReport> report,
+    UUID* uuid) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  base::FilePath path = ReportPath(report->ReportID(), kPending);
+  ScopedLockFile lock_file;
+  if (!lock_file.ResetAcquire(path)) {
+    return kBusyError;
+  }
+
+  if (!WriteNewMetadata(ReplaceFinalExtension(path, kMetadataExtension))) {
+    return kDatabaseError;
+  }
+
+  FileOffset size = report->Writer()->Seek(0, SEEK_END);
+
+  report->Writer()->Close();
+  if (!MoveFileOrDirectory(report->file_remover_.get(), path)) {
+    return kFileSystemError;
+  }
+  // We've moved the report to pending, so it no longer needs to be removed.
+  ignore_result(report->file_remover_.release());
+
+  *uuid = report->ReportID();
+
+  Metrics::CrashReportPending(Metrics::PendingReportReason::kNewlyCreated);
+  Metrics::CrashReportSize(size);
+
+  return kNoError;
+}
+
+OperationStatus CrashReportDatabaseGeneric::LookUpCrashReport(const UUID& uuid,
+                                                              Report* report) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  ScopedLockFile lock_file;
+  base::FilePath path;
+  return CheckoutReport(uuid, kSearchable, &path, &lock_file, report);
+}
+
+OperationStatus CrashReportDatabaseGeneric::GetPendingReports(
+    std::vector<Report>* reports) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+  return ReportsInState(kPending, reports);
+}
+
+OperationStatus CrashReportDatabaseGeneric::GetCompletedReports(
+    std::vector<Report>* reports) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+  return ReportsInState(kCompleted, reports);
+}
+
+OperationStatus CrashReportDatabaseGeneric::GetReportForUploading(
+    const UUID& uuid,
+    std::unique_ptr<const UploadReport>* report) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  auto upload_report = std::make_unique<LockfileUploadReport>();
+
+  base::FilePath path;
+  OperationStatus os = CheckoutReport(
+      uuid, kPending, &path, &upload_report->lock_file, upload_report.get());
+  if (os != kNoError) {
+    return os;
+  }
+
+  if (!upload_report->Initialize(path, this)) {
+    return kFileSystemError;
+  }
+
+  report->reset(upload_report.release());
+  return kNoError;
+}
+
+OperationStatus CrashReportDatabaseGeneric::SkipReportUpload(
+    const UUID& uuid,
+    Metrics::CrashSkippedReason reason) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  Metrics::CrashUploadSkipped(reason);
+
+  base::FilePath path;
+  ScopedLockFile lock_file;
+  Report report;
+  OperationStatus os =
+      CheckoutReport(uuid, kPending, &path, &lock_file, &report);
+  if (os != kNoError) {
+    return os;
+  }
+
+  base::FilePath completed_path(ReportPath(uuid, kCompleted));
+  ScopedLockFile completed_lock_file;
+  if (!completed_lock_file.ResetAcquire(completed_path)) {
+    return kBusyError;
+  }
+
+  report.upload_explicitly_requested = false;
+  if (!WriteMetadata(completed_path, report)) {
+    return kDatabaseError;
+  }
+
+  if (!MoveFileOrDirectory(path, completed_path)) {
+    return kFileSystemError;
+  }
+
+  if (!LoggingRemoveFile(ReplaceFinalExtension(path, kMetadataExtension))) {
+    return kDatabaseError;
+  }
+
+  return kNoError;
+}
+
+OperationStatus CrashReportDatabaseGeneric::DeleteReport(const UUID& uuid) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  base::FilePath path;
+  ScopedLockFile lock_file;
+  OperationStatus os =
+      LocateAndLockReport(uuid, kSearchable, &path, &lock_file);
+  if (os != kNoError) {
+    return os;
+  }
+
+  if (!LoggingRemoveFile(path)) {
+    return kFileSystemError;
+  }
+
+  if (!LoggingRemoveFile(ReplaceFinalExtension(path, kMetadataExtension))) {
+    return kDatabaseError;
+  }
+
+  return kNoError;
+}
+
+OperationStatus CrashReportDatabaseGeneric::RequestUpload(const UUID& uuid) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  base::FilePath path;
+  ScopedLockFile lock_file;
+  Report report;
+  OperationStatus os =
+      CheckoutReport(uuid, kSearchable, &path, &lock_file, &report);
+  if (os != kNoError) {
+    return os;
+  }
+
+  if (report.uploaded) {
+    return kCannotRequestUpload;
+  }
+
+  report.upload_explicitly_requested = true;
+  base::FilePath pending_path = ReportPath(uuid, kPending);
+  if (!MoveFileOrDirectory(path, pending_path)) {
+    return kFileSystemError;
+  }
+
+  if (!WriteMetadata(pending_path, report)) {
+    return kDatabaseError;
+  }
+
+  if (pending_path != path) {
+    if (!LoggingRemoveFile(ReplaceFinalExtension(path, kMetadataExtension))) {
+      return kDatabaseError;
+    }
+  }
+
+  Metrics::CrashReportPending(Metrics::PendingReportReason::kUserInitiated);
+  return kNoError;
+}
+
+int CrashReportDatabaseGeneric::CleanDatabase(time_t lockfile_ttl) {
+  int removed = 0;
+  time_t now = time(nullptr);
+
+  DirectoryReader reader;
+  const base::FilePath new_dir(base_dir_.Append(kNewDirectory));
+  if (reader.Open(new_dir)) {
+    base::FilePath filename;
+    DirectoryReader::Result result;
+    while ((result = reader.NextFile(&filename)) ==
+           DirectoryReader::Result::kSuccess) {
+      const base::FilePath filepath(new_dir.Append(filename));
+      timespec filetime;
+      if (!FileModificationTime(filepath, &filetime)) {
+        continue;
+      }
+      if (filetime.tv_sec <= now - lockfile_ttl) {
+        if (LoggingRemoveFile(filepath)) {
+          ++removed;
+        }
+      }
+    }
+  }
+
+  removed += CleanReportsInState(kPending, lockfile_ttl);
+  removed += CleanReportsInState(kCompleted, lockfile_ttl);
+  return removed;
+}
+
+OperationStatus CrashReportDatabaseGeneric::RecordUploadAttempt(
+    UploadReport* report,
+    bool successful,
+    const std::string& id) {
+  INITIALIZATION_STATE_DCHECK_VALID(initialized_);
+
+  Metrics::CrashUploadAttempted(successful);
+  time_t now = time(nullptr);
+
+  report->id = id;
+  report->uploaded = successful;
+  report->last_upload_attempt_time = now;
+  ++report->upload_attempts;
+
+  base::FilePath report_path(report->file_path);
+
+  ScopedLockFile lock_file;
+  if (successful) {
+    report->upload_explicitly_requested = false;
+
+    base::FilePath completed_report_path = ReportPath(report->uuid, kCompleted);
+
+    if (!lock_file.ResetAcquire(completed_report_path)) {
+      return kBusyError;
+    }
+
+    report->Reader()->Close();
+    if (!MoveFileOrDirectory(report_path, completed_report_path)) {
+      return kFileSystemError;
+    }
+
+    LoggingRemoveFile(ReplaceFinalExtension(report_path, kMetadataExtension));
+    report_path = completed_report_path;
+  }
+
+  if (!WriteMetadata(report_path, *report)) {
+    return kDatabaseError;
+  }
+
+  if (!settings_.SetLastUploadAttemptTime(now)) {
+    return kDatabaseError;
+  }
+
+  return kNoError;
+}
+
+base::FilePath CrashReportDatabaseGeneric::ReportPath(const UUID& uuid,
+                                                      ReportState state) {
+  DCHECK_NE(state, kUninitialized);
+  DCHECK_NE(state, kSearchable);
+
+#if defined(OS_WIN)
+  const std::wstring uuid_string = uuid.ToString16();
+#else
+  const std::string uuid_string = uuid.ToString();
+#endif
+
+  return base_dir_.Append(kReportDirectories[state])
+      .Append(uuid_string + kCrashReportExtension);
+}
+
+OperationStatus CrashReportDatabaseGeneric::LocateAndLockReport(
+    const UUID& uuid,
+    ReportState desired_state,
+    base::FilePath* path,
+    ScopedLockFile* lock_file) {
+  std::vector<ReportState> searchable_states;
+  if (desired_state == kSearchable) {
+    searchable_states.push_back(kPending);
+    searchable_states.push_back(kCompleted);
+  } else {
+    DCHECK(desired_state == kPending || desired_state == kCompleted);
+    searchable_states.push_back(desired_state);
+  }
+
+  for (const ReportState state : searchable_states) {
+    base::FilePath local_path(ReportPath(uuid, state));
+    ScopedLockFile local_lock;
+    if (!local_lock.ResetAcquire(local_path)) {
+      return kBusyError;
+    }
+
+    if (!IsRegularFile(local_path)) {
+      continue;
+    }
+
+    *path = local_path;
+    *lock_file = std::move(local_lock);
+    return kNoError;
+  }
+
+  return kReportNotFound;
+}
+
+OperationStatus CrashReportDatabaseGeneric::CheckoutReport(
+    const UUID& uuid,
+    ReportState state,
+    base::FilePath* path,
+    ScopedLockFile* lock_file,
+    Report* report) {
+  ScopedLockFile local_lock;
+  base::FilePath local_path;
+  OperationStatus os =
+      LocateAndLockReport(uuid, state, &local_path, &local_lock);
+  if (os != kNoError) {
+    return os;
+  }
+
+  if (!CleaningReadMetadata(local_path, report)) {
+    return kDatabaseError;
+  }
+
+  *path = local_path;
+  *lock_file = std::move(local_lock);
+  return kNoError;
+}
+
+OperationStatus CrashReportDatabaseGeneric::ReportsInState(
+    ReportState state,
+    std::vector<Report>* reports) {
+  DCHECK(reports->empty());
+  DCHECK_NE(state, kUninitialized);
+  DCHECK_NE(state, kSearchable);
+  DCHECK_NE(state, kNew);
+
+  const base::FilePath dir_path(base_dir_.Append(kReportDirectories[state]));
+  DirectoryReader reader;
+  if (!reader.Open(dir_path)) {
+    return kDatabaseError;
+  }
+
+  base::FilePath filename;
+  DirectoryReader::Result result;
+  while ((result = reader.NextFile(&filename)) ==
+         DirectoryReader::Result::kSuccess) {
+    const base::FilePath::StringType extension(filename.FinalExtension());
+    if (extension.compare(kCrashReportExtension) != 0) {
+      continue;
+    }
+
+    const base::FilePath filepath(dir_path.Append(filename));
+    ScopedLockFile lock_file;
+    if (!lock_file.ResetAcquire(filepath)) {
+      continue;
+    }
+
+    Report report;
+    if (!CleaningReadMetadata(filepath, &report)) {
+      continue;
+    }
+    reports->push_back(report);
+    reports->back().file_path = filepath;
+  }
+  return kNoError;
+}
+
+int CrashReportDatabaseGeneric::CleanReportsInState(ReportState state,
+                                                    time_t lockfile_ttl) {
+  const base::FilePath dir_path(base_dir_.Append(kReportDirectories[state]));
+  DirectoryReader reader;
+  if (!reader.Open(dir_path)) {
+    return 0;
+  }
+
+  int removed = 0;
+  base::FilePath filename;
+  DirectoryReader::Result result;
+  while ((result = reader.NextFile(&filename)) ==
+         DirectoryReader::Result::kSuccess) {
+    const base::FilePath::StringType extension(filename.FinalExtension());
+    const base::FilePath filepath(dir_path.Append(filename));
+
+    // Remove any report files without metadata.
+    if (extension.compare(kCrashReportExtension) == 0) {
+      const base::FilePath metadata_path(
+          ReplaceFinalExtension(filepath, kMetadataExtension));
+      ScopedLockFile report_lock;
+      if (report_lock.ResetAcquire(filepath) && !IsRegularFile(metadata_path) &&
+          LoggingRemoveFile(filepath)) {
+        ++removed;
+      }
+      continue;
+    }
+
+    // Remove any metadata files without report files.
+    if (extension.compare(kMetadataExtension) == 0) {
+      const base::FilePath report_path(
+          ReplaceFinalExtension(filepath, kCrashReportExtension));
+      ScopedLockFile report_lock;
+      if (report_lock.ResetAcquire(report_path) &&
+          !IsRegularFile(report_path) && LoggingRemoveFile(filepath)) {
+        ++removed;
+      }
+      continue;
+    }
+
+    // Remove any expired locks only if we can remove the report and metadata.
+    if (extension.compare(kLockExtension) == 0 &&
+        ScopedLockFile::IsExpired(filepath, lockfile_ttl)) {
+      const base::FilePath no_ext(filepath.RemoveFinalExtension());
+      const base::FilePath report_path(no_ext.value() + kCrashReportExtension);
+      const base::FilePath metadata_path(no_ext.value() + kMetadataExtension);
+      if ((IsRegularFile(report_path) && !LoggingRemoveFile(report_path)) ||
+          (IsRegularFile(metadata_path) && !LoggingRemoveFile(metadata_path))) {
+        continue;
+      }
+
+      if (LoggingRemoveFile(filepath)) {
+        ++removed;
+      }
+      continue;
+    }
+  }
+
+  return removed;
+}
+
+// static
+bool CrashReportDatabaseGeneric::ReadMetadata(const base::FilePath& path,
+                                              Report* report) {
+  const base::FilePath metadata_path(
+      ReplaceFinalExtension(path, kMetadataExtension));
+
+  ScopedFileHandle handle(LoggingOpenFileForRead(metadata_path));
+  if (!handle.is_valid()) {
+    return false;
+  }
+
+  if (!report->uuid.InitializeFromString(
+          path.BaseName().RemoveFinalExtension().value())) {
+    LOG(ERROR) << "Couldn't interpret report uuid";
+    return false;
+  }
+
+  ReportMetadata metadata;
+  if (!LoggingReadFileExactly(handle.get(), &metadata, sizeof(metadata))) {
+    return false;
+  }
+
+  if (metadata.version != ReportMetadata::kVersion) {
+    LOG(ERROR) << "metadata version mismatch";
+    return false;
+  }
+
+  if (!ReadRestOfFileAsString(handle.get(), &report->id)) {
+    return false;
+  }
+
+  report->upload_attempts = metadata.upload_attempts;
+  report->last_upload_attempt_time = metadata.last_upload_attempt_time;
+  report->creation_time = metadata.creation_time;
+  report->uploaded = (metadata.attributes & kAttributeUploaded) != 0;
+  report->upload_explicitly_requested =
+      (metadata.attributes & kAttributeUploadExplicitlyRequested) != 0;
+  report->file_path = path;
+  return true;
+}
+
+// static
+bool CrashReportDatabaseGeneric::CleaningReadMetadata(
+    const base::FilePath& path,
+    Report* report) {
+  if (ReadMetadata(path, report)) {
+    return true;
+  }
+
+  LoggingRemoveFile(path);
+  LoggingRemoveFile(ReplaceFinalExtension(path, kMetadataExtension));
+  return false;
+}
+
+// static
+bool CrashReportDatabaseGeneric::WriteNewMetadata(const base::FilePath& path) {
+  const base::FilePath metadata_path(
+      ReplaceFinalExtension(path, kMetadataExtension));
+
+  ScopedFileHandle handle(LoggingOpenFileForWrite(metadata_path,
+                                                  FileWriteMode::kCreateOrFail,
+                                                  FilePermissions::kOwnerOnly));
+  if (!handle.is_valid()) {
+    return false;
+  }
+
+  ReportMetadata metadata;
+  metadata.creation_time = time(nullptr);
+
+  return LoggingWriteFile(handle.get(), &metadata, sizeof(metadata));
+}
+
+// static
+bool CrashReportDatabaseGeneric::WriteMetadata(const base::FilePath& path,
+                                               const Report& report) {
+  const base::FilePath metadata_path(
+      ReplaceFinalExtension(path, kMetadataExtension));
+
+  ScopedFileHandle handle(
+      LoggingOpenFileForWrite(metadata_path,
+                              FileWriteMode::kTruncateOrCreate,
+                              FilePermissions::kOwnerOnly));
+  if (!handle.is_valid()) {
+    return false;
+  }
+
+  ReportMetadata metadata;
+  metadata.creation_time = report.creation_time;
+  metadata.last_upload_attempt_time = report.last_upload_attempt_time;
+  metadata.upload_attempts = report.upload_attempts;
+  metadata.attributes =
+      (report.uploaded ? kAttributeUploaded : 0) |
+      (report.upload_explicitly_requested ? kAttributeUploadExplicitlyRequested
+                                          : 0);
+
+  return LoggingWriteFile(handle.get(), &metadata, sizeof(metadata)) &&
+         LoggingWriteFile(handle.get(), report.id.c_str(), report.id.size());
+}
+
+}  // namespace crashpad

client/crash_report_database_test.cc

@@ -14,10 +14,12 @@
 
 #include "client/crash_report_database.h"
 
+#include "build/build_config.h"
 #include "client/settings.h"
 #include "gtest/gtest.h"
 #include "test/errors.h"
 #include "test/file.h"
+#include "test/filesystem.h"
 #include "test/scoped_temp_dir.h"
 #include "util/file/file_io.h"
 #include "util/file/filesystem.h"
@@ -667,6 +669,76 @@ TEST_F(CrashReportDatabaseTest, RequestUpload) {
             CrashReportDatabase::kCannotRequestUpload);
 }
 
+// This test uses knowledge of the database format to break it, so it only
+// applies to the unfified database implementation.
+#if !defined(OS_MACOSX) && !defined(OS_WIN)
+TEST_F(CrashReportDatabaseTest, CleanBrokenDatabase) {
+  // Remove report files if metadata goes missing.
+  CrashReportDatabase::Report report;
+  ASSERT_NO_FATAL_FAILURE(CreateCrashReport(&report));
+
+  const base::FilePath metadata(
+      report.file_path.RemoveFinalExtension().value() +
+      FILE_PATH_LITERAL(".meta"));
+  ASSERT_TRUE(PathExists(report.file_path));
+  ASSERT_TRUE(PathExists(metadata));
+
+  ASSERT_TRUE(LoggingRemoveFile(metadata));
+  EXPECT_EQ(db()->CleanDatabase(0), 1);
+
+  EXPECT_FALSE(PathExists(report.file_path));
+  EXPECT_FALSE(PathExists(metadata));
+
+  // Remove metadata files if reports go missing.
+  ASSERT_NO_FATAL_FAILURE(CreateCrashReport(&report));
+  const base::FilePath metadata2(
+      report.file_path.RemoveFinalExtension().value() +
+      FILE_PATH_LITERAL(".meta"));
+  ASSERT_TRUE(PathExists(report.file_path));
+  ASSERT_TRUE(PathExists(metadata2));
+
+  ASSERT_TRUE(LoggingRemoveFile(report.file_path));
+  EXPECT_EQ(db()->CleanDatabase(0), 1);
+
+  EXPECT_FALSE(PathExists(report.file_path));
+  EXPECT_FALSE(PathExists(metadata2));
+
+  // Remove stale new files.
+  std::unique_ptr<CrashReportDatabase::NewReport> new_report;
+  EXPECT_EQ(db()->PrepareNewCrashReport(&new_report),
+            CrashReportDatabase::kNoError);
+  new_report->Writer()->Close();
+  EXPECT_EQ(db()->CleanDatabase(0), 1);
+
+  // Remove stale lock files and their associated reports.
+  ASSERT_NO_FATAL_FAILURE(CreateCrashReport(&report));
+  const base::FilePath metadata3(
+      report.file_path.RemoveFinalExtension().value() +
+      FILE_PATH_LITERAL(".meta"));
+  ASSERT_TRUE(PathExists(report.file_path));
+  ASSERT_TRUE(PathExists(metadata3));
+
+  const base::FilePath lockpath(
+      report.file_path.RemoveFinalExtension().value() +
+      FILE_PATH_LITERAL(".lock"));
+  ScopedFileHandle handle(LoggingOpenFileForWrite(
+      lockpath, FileWriteMode::kCreateOrFail, FilePermissions::kOwnerOnly));
+  ASSERT_TRUE(handle.is_valid());
+
+  time_t expired_timestamp = time(nullptr) - 60 * 60 * 24 * 3;
+
+  ASSERT_TRUE(LoggingWriteFile(
+      handle.get(), &expired_timestamp, sizeof(expired_timestamp)));
+  ASSERT_TRUE(LoggingCloseFile(handle.get()));
+  ignore_result(handle.release());
+
+  EXPECT_EQ(db()->CleanDatabase(0), 1);
+
+  EXPECT_FALSE(PathExists(report.file_path));
+  EXPECT_FALSE(PathExists(metadata3));
+}
+#endif  // !OS_MACOSX && !OS_WIN
+
 }  // namespace
 }  // namespace test
 }  // namespace crashpad

handler/prune_crash_reports_thread.cc

@@ -38,6 +38,7 @@ void PruneCrashReportThread::Stop() {
 }
 
 void PruneCrashReportThread::DoWork(const WorkerThread* thread) {
+  database_->CleanDatabase(60 * 60 * 24 * 3);
   PruneCrashReportDatabase(database_, condition_.get());
 }