Compare commits
1 Commits
main
...
doc-testin
Author | SHA1 | Date | |
---|---|---|---|
|
031ae8e2db |
35
.appveyor.yml
Normal file
35
.appveyor.yml
Normal file
@ -0,0 +1,35 @@
|
||||
# Build matrix / environment variables are explained on:
|
||||
# https://www.appveyor.com/docs/appveyor-yml/
|
||||
# This file can be validated on: https://ci.appveyor.com/tools/validate-yaml
|
||||
|
||||
version: "{build}"
|
||||
|
||||
environment:
|
||||
matrix:
|
||||
# AppVeyor currently has no custom job name feature.
|
||||
# http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs
|
||||
- JOB: Visual Studio 2017
|
||||
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
|
||||
CMAKE_GENERATOR: Visual Studio 15 2017
|
||||
|
||||
platform:
|
||||
- x86
|
||||
- x64
|
||||
|
||||
configuration:
|
||||
- RelWithDebInfo
|
||||
- Debug
|
||||
|
||||
build_script:
|
||||
- git submodule update --init --recursive
|
||||
- mkdir build
|
||||
- cd build
|
||||
- if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64
|
||||
- cmake --version
|
||||
- cmake .. -G "%CMAKE_GENERATOR%"
|
||||
-DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%"
|
||||
- cmake --build . --config "%CONFIGURATION%"
|
||||
- cd ..
|
||||
|
||||
test_script:
|
||||
- cd build && ctest --verbose --build-config "%CONFIGURATION%" && cd ..
|
@ -1,18 +0,0 @@
|
||||
# Run manually to reformat a file:
|
||||
# clang-format -i --style=file <file>
|
||||
# find . -iname '*.cc' -o -iname '*.h' -o -iname '*.h.in' | xargs clang-format -i --style=file
|
||||
BasedOnStyle: Google
|
||||
DerivePointerAlignment: false
|
||||
|
||||
# Public headers are in a different location in the internal Google repository.
|
||||
# Order them so that when imported to the authoritative repository they will be
|
||||
# in correct alphabetical order.
|
||||
IncludeCategories:
|
||||
- Regex: '^(<|"(benchmarks|db|helpers)/)'
|
||||
Priority: 1
|
||||
- Regex: '^"(leveldb)/'
|
||||
Priority: 2
|
||||
- Regex: '^(<|"(issues|port|table|third_party|util)/)'
|
||||
Priority: 3
|
||||
- Regex: '.*'
|
||||
Priority: 4
|
102
.github/workflows/build.yml
vendored
102
.github/workflows/build.yml
vendored
@ -1,102 +0,0 @@
|
||||
# Copyright 2021 The LevelDB Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
name: ci
|
||||
on: [push, pull_request]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-and-test:
|
||||
name: >-
|
||||
CI
|
||||
${{ matrix.os }}
|
||||
${{ matrix.compiler }}
|
||||
${{ matrix.optimized && 'release' || 'debug' }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
compiler: [clang, gcc, msvc]
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
optimized: [true, false]
|
||||
exclude:
|
||||
# MSVC only works on Windows.
|
||||
- os: ubuntu-latest
|
||||
compiler: msvc
|
||||
- os: macos-latest
|
||||
compiler: msvc
|
||||
# Not testing with GCC on macOS.
|
||||
- os: macos-latest
|
||||
compiler: gcc
|
||||
# Only testing with MSVC on Windows.
|
||||
- os: windows-latest
|
||||
compiler: clang
|
||||
- os: windows-latest
|
||||
compiler: gcc
|
||||
include:
|
||||
- compiler: clang
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
- compiler: gcc
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
- compiler: msvc
|
||||
CC:
|
||||
CXX:
|
||||
|
||||
env:
|
||||
CMAKE_BUILD_DIR: ${{ github.workspace }}/build
|
||||
CMAKE_BUILD_TYPE: ${{ matrix.optimized && 'RelWithDebInfo' || 'Debug' }}
|
||||
CC: ${{ matrix.CC }}
|
||||
CXX: ${{ matrix.CXX }}
|
||||
BINARY_SUFFIX: ${{ startsWith(matrix.os, 'windows') && '.exe' || '' }}
|
||||
BINARY_PATH: >-
|
||||
${{ format(
|
||||
startsWith(matrix.os, 'windows') && '{0}\build\{1}\' || '{0}/build/',
|
||||
github.workspace,
|
||||
matrix.optimized && 'RelWithDebInfo' || 'Debug') }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Install dependencies on Linux
|
||||
if: ${{ runner.os == 'Linux' }}
|
||||
# libgoogle-perftools-dev is temporarily removed from the package list
|
||||
# because it is currently broken on GitHub's Ubuntu 22.04.
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libkyotocabinet-dev libsnappy-dev libsqlite3-dev
|
||||
|
||||
- name: Generate build config
|
||||
run: >-
|
||||
cmake -S "${{ github.workspace }}" -B "${{ env.CMAKE_BUILD_DIR }}"
|
||||
-DCMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }}
|
||||
-DCMAKE_INSTALL_PREFIX=${{ runner.temp }}/install_test/
|
||||
|
||||
- name: Build
|
||||
run: >-
|
||||
cmake --build "${{ env.CMAKE_BUILD_DIR }}"
|
||||
--config "${{ env.CMAKE_BUILD_TYPE }}"
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: ${{ github.workspace }}/build
|
||||
run: ctest -C "${{ env.CMAKE_BUILD_TYPE }}" --verbose
|
||||
|
||||
- name: Run LevelDB Benchmarks
|
||||
run: ${{ env.BINARY_PATH }}db_bench${{ env.BINARY_SUFFIX }}
|
||||
|
||||
- name: Run SQLite Benchmarks
|
||||
if: ${{ runner.os != 'Windows' }}
|
||||
run: ${{ env.BINARY_PATH }}db_bench_sqlite3${{ env.BINARY_SUFFIX }}
|
||||
|
||||
- name: Run Kyoto Cabinet Benchmarks
|
||||
if: ${{ runner.os == 'Linux' && matrix.compiler == 'clang' }}
|
||||
run: ${{ env.BINARY_PATH }}db_bench_tree_db${{ env.BINARY_SUFFIX }}
|
||||
|
||||
- name: Test CMake installation
|
||||
run: cmake --build "${{ env.CMAKE_BUILD_DIR }}" --target install
|
6
.gitmodules
vendored
6
.gitmodules
vendored
@ -1,6 +0,0 @@
|
||||
[submodule "third_party/googletest"]
|
||||
path = third_party/googletest
|
||||
url = https://github.com/google/googletest.git
|
||||
[submodule "third_party/benchmark"]
|
||||
path = third_party/benchmark
|
||||
url = https://github.com/google/benchmark
|
76
.travis.yml
Normal file
76
.travis.yml
Normal file
@ -0,0 +1,76 @@
|
||||
# Build matrix / environment variable are explained on:
|
||||
# http://about.travis-ci.org/docs/user/build-configuration/
|
||||
# This file can be validated on: http://lint.travis-ci.org/
|
||||
|
||||
dist: xenial
|
||||
language: cpp
|
||||
|
||||
compiler:
|
||||
- gcc
|
||||
- clang
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
|
||||
env:
|
||||
- BUILD_TYPE=Debug
|
||||
- BUILD_TYPE=RelWithDebInfo
|
||||
|
||||
matrix:
|
||||
exclude:
|
||||
# GCC fails on recent Travis OSX images.
|
||||
# https://github.com/travis-ci/travis-ci/issues/9640
|
||||
- compiler: gcc
|
||||
os: osx
|
||||
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- llvm-toolchain-xenial-7
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- clang-7
|
||||
- cmake
|
||||
- gcc-8
|
||||
- g++-8
|
||||
- libgoogle-perftools-dev
|
||||
- libkyotocabinet-dev
|
||||
- libsnappy-dev
|
||||
- libsqlite3-dev
|
||||
- ninja-build
|
||||
homebrew:
|
||||
packages:
|
||||
- crc32c
|
||||
- gperftools
|
||||
- kyotocabinet
|
||||
- gcc@7
|
||||
- ninja
|
||||
- snappy
|
||||
- sqlite3
|
||||
|
||||
before_install:
|
||||
# The Travis VM image for Mac already has a link at /usr/local/include/c++,
|
||||
# causing Homebrew's gcc installation to error out. This was reported to
|
||||
# Homebrew maintainers at https://github.com/Homebrew/brew/issues/1742 and
|
||||
# removing the link emerged as a workaround.
|
||||
- if [ "$TRAVIS_OS_NAME" == "osx" ]; then rm -f /usr/local/include/c++ ; fi
|
||||
|
||||
install:
|
||||
# /usr/bin/gcc is stuck to old versions on both Linux and OSX.
|
||||
- if [ "$CXX" = "g++" ]; then export CXX="g++-8" CC="gcc-8"; fi
|
||||
- echo ${CC}
|
||||
- echo ${CXX}
|
||||
- ${CXX} --version
|
||||
- cmake --version
|
||||
|
||||
before_script:
|
||||
- mkdir -p build && cd build
|
||||
- cmake .. -G Ninja -DCMAKE_BUILD_TYPE=$BUILD_TYPE
|
||||
- cmake --build .
|
||||
- cd ..
|
||||
|
||||
script:
|
||||
- cd build && ctest --verbose && cd ..
|
||||
- "if [ -f build/db_bench ] ; then build/db_bench ; fi"
|
||||
- "if [ -f build/db_bench_sqlite3 ] ; then build/db_bench_sqlite3 ; fi"
|
||||
- "if [ -f build/db_bench_tree_db ] ; then build/db_bench_tree_db ; fi"
|
432
CMakeLists.txt
432
CMakeLists.txt
@ -4,23 +4,17 @@
|
||||
|
||||
cmake_minimum_required(VERSION 3.9)
|
||||
# Keep the version below in sync with the one in db.h
|
||||
project(leveldb VERSION 1.23.0 LANGUAGES C CXX)
|
||||
project(leveldb VERSION 1.21.0 LANGUAGES C CXX)
|
||||
|
||||
# C standard can be overridden when this is used as a sub-project.
|
||||
if(NOT CMAKE_C_STANDARD)
|
||||
# This project can use C11, but will gracefully decay down to C89.
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED OFF)
|
||||
set(CMAKE_C_EXTENSIONS OFF)
|
||||
endif(NOT CMAKE_C_STANDARD)
|
||||
# This project can use C11, but will gracefully decay down to C89.
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED OFF)
|
||||
set(CMAKE_C_EXTENSIONS OFF)
|
||||
|
||||
# C++ standard can be overridden when this is used as a sub-project.
|
||||
if(NOT CMAKE_CXX_STANDARD)
|
||||
# This project requires C++11.
|
||||
set(CMAKE_CXX_STANDARD 11)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
endif(NOT CMAKE_CXX_STANDARD)
|
||||
# This project requires C++11.
|
||||
set(CMAKE_CXX_STANDARD 11)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
|
||||
if (WIN32)
|
||||
set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_WINDOWS)
|
||||
@ -34,13 +28,15 @@ option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" ON)
|
||||
option(LEVELDB_BUILD_BENCHMARKS "Build LevelDB's benchmarks" ON)
|
||||
option(LEVELDB_INSTALL "Install LevelDB's header and library" ON)
|
||||
|
||||
include(TestBigEndian)
|
||||
test_big_endian(LEVELDB_IS_BIG_ENDIAN)
|
||||
|
||||
include(CheckIncludeFile)
|
||||
check_include_file("unistd.h" HAVE_UNISTD_H)
|
||||
|
||||
include(CheckLibraryExists)
|
||||
check_library_exists(crc32c crc32c_value "" HAVE_CRC32C)
|
||||
check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)
|
||||
check_library_exists(zstd zstd_compress "" HAVE_ZSTD)
|
||||
check_library_exists(tcmalloc malloc "" HAVE_TCMALLOC)
|
||||
|
||||
include(CheckCXXSymbolExists)
|
||||
@ -51,42 +47,26 @@ include(CheckCXXSymbolExists)
|
||||
# (-std=c11), but do expose the function in standard C++ mode (-std=c++11).
|
||||
check_cxx_symbol_exists(fdatasync "unistd.h" HAVE_FDATASYNC)
|
||||
check_cxx_symbol_exists(F_FULLFSYNC "fcntl.h" HAVE_FULLFSYNC)
|
||||
check_cxx_symbol_exists(O_CLOEXEC "fcntl.h" HAVE_O_CLOEXEC)
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
||||
# Disable C++ exceptions.
|
||||
string(REGEX REPLACE "/EH[a-z]+" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
|
||||
add_definitions(-D_HAS_EXCEPTIONS=0)
|
||||
|
||||
# Disable RTTI.
|
||||
string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
|
||||
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
||||
# Enable strict prototype warnings for C code in clang and gcc.
|
||||
if(NOT CMAKE_C_FLAGS MATCHES "-Wstrict-prototypes")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes")
|
||||
endif(NOT CMAKE_C_FLAGS MATCHES "-Wstrict-prototypes")
|
||||
|
||||
# Disable C++ exceptions.
|
||||
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
|
||||
|
||||
# Disable RTTI.
|
||||
string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
|
||||
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
|
||||
include(CheckCXXSourceCompiles)
|
||||
|
||||
# Test whether -Wthread-safety is available. See
|
||||
# https://clang.llvm.org/docs/ThreadSafetyAnalysis.html
|
||||
include(CheckCXXCompilerFlag)
|
||||
check_cxx_compiler_flag(-Wthread-safety HAVE_CLANG_THREAD_SAFETY)
|
||||
|
||||
# Used by googletest.
|
||||
check_cxx_compiler_flag(-Wno-missing-field-initializers
|
||||
LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
|
||||
|
||||
include(CheckCXXSourceCompiles)
|
||||
# -Werror is necessary because unknown attributes only generate warnings.
|
||||
set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
|
||||
list(APPEND CMAKE_REQUIRED_FLAGS -Werror -Wthread-safety)
|
||||
check_cxx_source_compiles("
|
||||
struct __attribute__((lockable)) Lock {
|
||||
void Acquire() __attribute__((exclusive_lock_function()));
|
||||
void Release() __attribute__((unlock_function()));
|
||||
};
|
||||
struct ThreadSafeType {
|
||||
Lock lock_;
|
||||
int data_ __attribute__((guarded_by(lock_)));
|
||||
};
|
||||
int main() { return 0; }
|
||||
" HAVE_CLANG_THREAD_SAFETY)
|
||||
set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
|
||||
|
||||
# Test whether C++17 __has_include is available.
|
||||
check_cxx_source_compiles("
|
||||
@ -100,13 +80,13 @@ set(LEVELDB_PUBLIC_INCLUDE_DIR "include/leveldb")
|
||||
set(LEVELDB_PORT_CONFIG_DIR "include/port")
|
||||
|
||||
configure_file(
|
||||
"port/port_config.h.in"
|
||||
"${PROJECT_SOURCE_DIR}/port/port_config.h.in"
|
||||
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
|
||||
)
|
||||
|
||||
include_directories(
|
||||
"${PROJECT_BINARY_DIR}/include"
|
||||
"."
|
||||
"${PROJECT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(BUILD_SHARED_LIBS)
|
||||
@ -114,82 +94,79 @@ if(BUILD_SHARED_LIBS)
|
||||
add_compile_options(-fvisibility=hidden)
|
||||
endif(BUILD_SHARED_LIBS)
|
||||
|
||||
# Must be included before CMAKE_INSTALL_INCLUDEDIR is used.
|
||||
include(GNUInstallDirs)
|
||||
|
||||
add_library(leveldb "")
|
||||
target_sources(leveldb
|
||||
PRIVATE
|
||||
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
|
||||
"db/builder.cc"
|
||||
"db/builder.h"
|
||||
"db/c.cc"
|
||||
"db/db_impl.cc"
|
||||
"db/db_impl.h"
|
||||
"db/db_iter.cc"
|
||||
"db/db_iter.h"
|
||||
"db/dbformat.cc"
|
||||
"db/dbformat.h"
|
||||
"db/dumpfile.cc"
|
||||
"db/filename.cc"
|
||||
"db/filename.h"
|
||||
"db/log_format.h"
|
||||
"db/log_reader.cc"
|
||||
"db/log_reader.h"
|
||||
"db/log_writer.cc"
|
||||
"db/log_writer.h"
|
||||
"db/memtable.cc"
|
||||
"db/memtable.h"
|
||||
"db/repair.cc"
|
||||
"db/skiplist.h"
|
||||
"db/snapshot.h"
|
||||
"db/table_cache.cc"
|
||||
"db/table_cache.h"
|
||||
"db/version_edit.cc"
|
||||
"db/version_edit.h"
|
||||
"db/version_set.cc"
|
||||
"db/version_set.h"
|
||||
"db/write_batch_internal.h"
|
||||
"db/write_batch.cc"
|
||||
"port/port_stdcxx.h"
|
||||
"port/port.h"
|
||||
"port/thread_annotations.h"
|
||||
"table/block_builder.cc"
|
||||
"table/block_builder.h"
|
||||
"table/block.cc"
|
||||
"table/block.h"
|
||||
"table/filter_block.cc"
|
||||
"table/filter_block.h"
|
||||
"table/format.cc"
|
||||
"table/format.h"
|
||||
"table/iterator_wrapper.h"
|
||||
"table/iterator.cc"
|
||||
"table/merger.cc"
|
||||
"table/merger.h"
|
||||
"table/table_builder.cc"
|
||||
"table/table.cc"
|
||||
"table/two_level_iterator.cc"
|
||||
"table/two_level_iterator.h"
|
||||
"util/arena.cc"
|
||||
"util/arena.h"
|
||||
"util/bloom.cc"
|
||||
"util/cache.cc"
|
||||
"util/coding.cc"
|
||||
"util/coding.h"
|
||||
"util/comparator.cc"
|
||||
"util/crc32c.cc"
|
||||
"util/crc32c.h"
|
||||
"util/env.cc"
|
||||
"util/filter_policy.cc"
|
||||
"util/hash.cc"
|
||||
"util/hash.h"
|
||||
"util/logging.cc"
|
||||
"util/logging.h"
|
||||
"util/mutexlock.h"
|
||||
"util/no_destructor.h"
|
||||
"util/options.cc"
|
||||
"util/random.h"
|
||||
"util/status.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/builder.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/builder.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/c.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/db_impl.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/db_impl.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/db_iter.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/db_iter.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/dbformat.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/dbformat.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/dumpfile.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/filename.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/filename.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/log_format.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/log_reader.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/log_reader.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/log_writer.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/log_writer.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/memtable.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/memtable.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/repair.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/skiplist.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/snapshot.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/table_cache.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/table_cache.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/version_edit.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/version_edit.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/version_set.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/version_set.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/write_batch_internal.h"
|
||||
"${PROJECT_SOURCE_DIR}/db/write_batch.cc"
|
||||
"${PROJECT_SOURCE_DIR}/port/port_stdcxx.h"
|
||||
"${PROJECT_SOURCE_DIR}/port/port.h"
|
||||
"${PROJECT_SOURCE_DIR}/port/thread_annotations.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/block_builder.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/block_builder.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/block.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/block.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/filter_block.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/filter_block.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/format.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/format.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/iterator_wrapper.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/iterator.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/merger.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/merger.h"
|
||||
"${PROJECT_SOURCE_DIR}/table/table_builder.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/table.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/two_level_iterator.cc"
|
||||
"${PROJECT_SOURCE_DIR}/table/two_level_iterator.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/arena.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/arena.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/bloom.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/cache.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/coding.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/coding.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/comparator.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/crc32c.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/crc32c.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/env.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/filter_policy.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/hash.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/hash.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/logging.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/logging.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/mutexlock.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/no_destructor.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/options.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/random.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/status.cc"
|
||||
|
||||
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
|
||||
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
|
||||
@ -213,22 +190,22 @@ target_sources(leveldb
|
||||
if (WIN32)
|
||||
target_sources(leveldb
|
||||
PRIVATE
|
||||
"util/env_windows.cc"
|
||||
"util/windows_logger.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/env_windows.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/windows_logger.h"
|
||||
)
|
||||
else (WIN32)
|
||||
target_sources(leveldb
|
||||
PRIVATE
|
||||
"util/env_posix.cc"
|
||||
"util/posix_logger.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/env_posix.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/posix_logger.h"
|
||||
)
|
||||
endif (WIN32)
|
||||
|
||||
# MemEnv is not part of the interface and could be pulled to a separate library.
|
||||
target_sources(leveldb
|
||||
PRIVATE
|
||||
"helpers/memenv/memenv.cc"
|
||||
"helpers/memenv/memenv.h"
|
||||
"${PROJECT_SOURCE_DIR}/helpers/memenv/memenv.cc"
|
||||
"${PROJECT_SOURCE_DIR}/helpers/memenv/memenv.h"
|
||||
)
|
||||
|
||||
target_include_directories(leveldb
|
||||
@ -274,9 +251,6 @@ endif(HAVE_CRC32C)
|
||||
if(HAVE_SNAPPY)
|
||||
target_link_libraries(leveldb snappy)
|
||||
endif(HAVE_SNAPPY)
|
||||
if(HAVE_ZSTD)
|
||||
target_link_libraries(leveldb zstd)
|
||||
endif(HAVE_ZSTD)
|
||||
if(HAVE_TCMALLOC)
|
||||
target_link_libraries(leveldb tcmalloc)
|
||||
endif(HAVE_TCMALLOC)
|
||||
@ -286,84 +260,13 @@ find_package(Threads REQUIRED)
|
||||
target_link_libraries(leveldb Threads::Threads)
|
||||
|
||||
add_executable(leveldbutil
|
||||
"db/leveldbutil.cc"
|
||||
"${PROJECT_SOURCE_DIR}/db/leveldbutil.cc"
|
||||
)
|
||||
target_link_libraries(leveldbutil leveldb)
|
||||
|
||||
if(LEVELDB_BUILD_TESTS)
|
||||
enable_testing()
|
||||
|
||||
# Prevent overriding the parent project's compiler/linker settings on Windows.
|
||||
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
|
||||
set(install_gtest OFF)
|
||||
set(install_gmock OFF)
|
||||
set(build_gmock ON)
|
||||
|
||||
# This project is tested using GoogleTest.
|
||||
add_subdirectory("third_party/googletest")
|
||||
|
||||
# GoogleTest triggers a missing field initializers warning.
|
||||
if(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
|
||||
set_property(TARGET gtest
|
||||
APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers)
|
||||
set_property(TARGET gmock
|
||||
APPEND PROPERTY COMPILE_OPTIONS -Wno-missing-field-initializers)
|
||||
endif(LEVELDB_HAVE_NO_MISSING_FIELD_INITIALIZERS)
|
||||
|
||||
add_executable(leveldb_tests "")
|
||||
target_sources(leveldb_tests
|
||||
PRIVATE
|
||||
# "db/fault_injection_test.cc"
|
||||
# "issues/issue178_test.cc"
|
||||
# "issues/issue200_test.cc"
|
||||
# "issues/issue320_test.cc"
|
||||
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
|
||||
# "util/env_test.cc"
|
||||
"util/status_test.cc"
|
||||
"util/no_destructor_test.cc"
|
||||
"util/testutil.cc"
|
||||
"util/testutil.h"
|
||||
)
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
target_sources(leveldb_tests
|
||||
PRIVATE
|
||||
"db/autocompact_test.cc"
|
||||
"db/corruption_test.cc"
|
||||
"db/db_test.cc"
|
||||
"db/dbformat_test.cc"
|
||||
"db/filename_test.cc"
|
||||
"db/log_test.cc"
|
||||
"db/recovery_test.cc"
|
||||
"db/skiplist_test.cc"
|
||||
"db/version_edit_test.cc"
|
||||
"db/version_set_test.cc"
|
||||
"db/write_batch_test.cc"
|
||||
"helpers/memenv/memenv_test.cc"
|
||||
"table/filter_block_test.cc"
|
||||
"table/table_test.cc"
|
||||
"util/arena_test.cc"
|
||||
"util/bloom_test.cc"
|
||||
"util/cache_test.cc"
|
||||
"util/coding_test.cc"
|
||||
"util/crc32c_test.cc"
|
||||
"util/hash_test.cc"
|
||||
"util/logging_test.cc"
|
||||
)
|
||||
endif(NOT BUILD_SHARED_LIBS)
|
||||
target_link_libraries(leveldb_tests leveldb gmock gtest gtest_main)
|
||||
target_compile_definitions(leveldb_tests
|
||||
PRIVATE
|
||||
${LEVELDB_PLATFORM_NAME}=1
|
||||
)
|
||||
if (NOT HAVE_CXX17_HAS_INCLUDE)
|
||||
target_compile_definitions(leveldb_tests
|
||||
PRIVATE
|
||||
LEVELDB_HAS_PORT_CONFIG_H=1
|
||||
)
|
||||
endif(NOT HAVE_CXX17_HAS_INCLUDE)
|
||||
|
||||
add_test(NAME "leveldb_tests" COMMAND "leveldb_tests")
|
||||
|
||||
function(leveldb_test test_file)
|
||||
get_filename_component(test_target_name "${test_file}" NAME_WE)
|
||||
|
||||
@ -371,12 +274,14 @@ if(LEVELDB_BUILD_TESTS)
|
||||
target_sources("${test_target_name}"
|
||||
PRIVATE
|
||||
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
|
||||
"util/testutil.cc"
|
||||
"util/testutil.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/testharness.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/testharness.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/testutil.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/testutil.h"
|
||||
|
||||
"${test_file}"
|
||||
)
|
||||
target_link_libraries("${test_target_name}" leveldb gmock gtest)
|
||||
target_link_libraries("${test_target_name}" leveldb)
|
||||
target_compile_definitions("${test_target_name}"
|
||||
PRIVATE
|
||||
${LEVELDB_PLATFORM_NAME}=1
|
||||
@ -391,25 +296,54 @@ if(LEVELDB_BUILD_TESTS)
|
||||
add_test(NAME "${test_target_name}" COMMAND "${test_target_name}")
|
||||
endfunction(leveldb_test)
|
||||
|
||||
leveldb_test("db/c_test.c")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/c_test.c")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/fault_injection_test.cc")
|
||||
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/issues/issue178_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/issues/issue200_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/issues/issue320_test.cc")
|
||||
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/env_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/status_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/no_destructor_test.cc")
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/autocompact_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/corruption_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/db_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/dbformat_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/filename_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/log_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/recovery_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/skiplist_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/version_edit_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/version_set_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/db/write_batch_test.cc")
|
||||
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/helpers/memenv/memenv_test.cc")
|
||||
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/table/filter_block_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/table/table_test.cc")
|
||||
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/arena_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/bloom_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/cache_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/coding_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/crc32c_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/hash_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/logging_test.cc")
|
||||
|
||||
# TODO(costan): This test also uses
|
||||
# "util/env_{posix|windows}_test_helper.h"
|
||||
# "${PROJECT_SOURCE_DIR}/util/env_{posix|windows}_test_helper.h"
|
||||
if (WIN32)
|
||||
leveldb_test("util/env_windows_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/env_windows_test.cc")
|
||||
else (WIN32)
|
||||
leveldb_test("util/env_posix_test.cc")
|
||||
leveldb_test("${PROJECT_SOURCE_DIR}/util/env_posix_test.cc")
|
||||
endif (WIN32)
|
||||
endif(NOT BUILD_SHARED_LIBS)
|
||||
endif(LEVELDB_BUILD_TESTS)
|
||||
|
||||
if(LEVELDB_BUILD_BENCHMARKS)
|
||||
# This project uses Google benchmark for benchmarking.
|
||||
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
|
||||
set(BENCHMARK_ENABLE_EXCEPTIONS OFF CACHE BOOL "" FORCE)
|
||||
add_subdirectory("third_party/benchmark")
|
||||
|
||||
function(leveldb_benchmark bench_file)
|
||||
get_filename_component(bench_target_name "${bench_file}" NAME_WE)
|
||||
|
||||
@ -417,14 +351,16 @@ if(LEVELDB_BUILD_BENCHMARKS)
|
||||
target_sources("${bench_target_name}"
|
||||
PRIVATE
|
||||
"${PROJECT_BINARY_DIR}/${LEVELDB_PORT_CONFIG_DIR}/port_config.h"
|
||||
"util/histogram.cc"
|
||||
"util/histogram.h"
|
||||
"util/testutil.cc"
|
||||
"util/testutil.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/histogram.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/histogram.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/testharness.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/testharness.h"
|
||||
"${PROJECT_SOURCE_DIR}/util/testutil.cc"
|
||||
"${PROJECT_SOURCE_DIR}/util/testutil.h"
|
||||
|
||||
"${bench_file}"
|
||||
)
|
||||
target_link_libraries("${bench_target_name}" leveldb gmock gtest benchmark)
|
||||
target_link_libraries("${bench_target_name}" leveldb)
|
||||
target_compile_definitions("${bench_target_name}"
|
||||
PRIVATE
|
||||
${LEVELDB_PLATFORM_NAME}=1
|
||||
@ -438,12 +374,12 @@ if(LEVELDB_BUILD_BENCHMARKS)
|
||||
endfunction(leveldb_benchmark)
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
leveldb_benchmark("benchmarks/db_bench.cc")
|
||||
leveldb_benchmark("${PROJECT_SOURCE_DIR}/db/db_bench.cc")
|
||||
endif(NOT BUILD_SHARED_LIBS)
|
||||
|
||||
check_library_exists(sqlite3 sqlite3_open "" HAVE_SQLITE3)
|
||||
if(HAVE_SQLITE3)
|
||||
leveldb_benchmark("benchmarks/db_bench_sqlite3.cc")
|
||||
leveldb_benchmark("${PROJECT_SOURCE_DIR}/doc/bench/db_bench_sqlite3.cc")
|
||||
target_link_libraries(db_bench_sqlite3 sqlite3)
|
||||
endif(HAVE_SQLITE3)
|
||||
|
||||
@ -463,12 +399,13 @@ int main() {
|
||||
" HAVE_KYOTOCABINET)
|
||||
set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQURED_LIBRARIES})
|
||||
if(HAVE_KYOTOCABINET)
|
||||
leveldb_benchmark("benchmarks/db_bench_tree_db.cc")
|
||||
leveldb_benchmark("${PROJECT_SOURCE_DIR}/doc/bench/db_bench_tree_db.cc")
|
||||
target_link_libraries(db_bench_tree_db kyotocabinet)
|
||||
endif(HAVE_KYOTOCABINET)
|
||||
endif(LEVELDB_BUILD_BENCHMARKS)
|
||||
|
||||
if(LEVELDB_INSTALL)
|
||||
include(GNUInstallDirs)
|
||||
install(TARGETS leveldb
|
||||
EXPORT leveldbTargets
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
@ -477,43 +414,38 @@ if(LEVELDB_INSTALL)
|
||||
)
|
||||
install(
|
||||
FILES
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/c.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/cache.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/comparator.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/db.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/dumpfile.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/env.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/export.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/filter_policy.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/iterator.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/options.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/slice.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/status.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/table_builder.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/table.h"
|
||||
"${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h"
|
||||
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/leveldb"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/c.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/cache.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/comparator.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/db.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/dumpfile.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/env.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/export.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/filter_policy.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/iterator.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/options.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/slice.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/status.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/table_builder.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/table.h"
|
||||
"${PROJECT_SOURCE_DIR}/${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h"
|
||||
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/leveldb
|
||||
)
|
||||
|
||||
include(CMakePackageConfigHelpers)
|
||||
configure_package_config_file(
|
||||
"cmake/${PROJECT_NAME}Config.cmake.in"
|
||||
"${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake"
|
||||
INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}"
|
||||
)
|
||||
write_basic_package_version_file(
|
||||
"${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake"
|
||||
COMPATIBILITY SameMajorVersion
|
||||
"${PROJECT_BINARY_DIR}/leveldbConfigVersion.cmake"
|
||||
COMPATIBILITY SameMajorVersion
|
||||
)
|
||||
install(
|
||||
EXPORT leveldbTargets
|
||||
NAMESPACE leveldb::
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/leveldb"
|
||||
)
|
||||
install(
|
||||
FILES
|
||||
"${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}Config.cmake"
|
||||
"${PROJECT_BINARY_DIR}/cmake/${PROJECT_NAME}ConfigVersion.cmake"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}"
|
||||
"${PROJECT_SOURCE_DIR}/cmake/leveldbConfig.cmake"
|
||||
"${PROJECT_BINARY_DIR}/leveldbConfigVersion.cmake"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/leveldb"
|
||||
)
|
||||
endif(LEVELDB_INSTALL)
|
||||
|
@ -1,31 +1,36 @@
|
||||
# How to Contribute
|
||||
# Contributing
|
||||
|
||||
We'd love to accept your patches and contributions to this project. There are
|
||||
just a few small guidelines you need to follow.
|
||||
We'd love to accept your code patches! However, before we can take them, we
|
||||
have to jump a couple of legal hurdles.
|
||||
|
||||
## Contributor License Agreement
|
||||
## Contributor License Agreements
|
||||
|
||||
Contributions to this project must be accompanied by a Contributor License
|
||||
Agreement. You (or your employer) retain the copyright to your contribution;
|
||||
this simply gives us permission to use and redistribute your contributions as
|
||||
part of the project. Head over to <https://cla.developers.google.com/> to see
|
||||
your current agreements on file or to sign a new one.
|
||||
Please fill out either the individual or corporate Contributor License
|
||||
Agreement as appropriate.
|
||||
|
||||
You generally only need to submit a CLA once, so if you've already submitted one
|
||||
(even if it was for a different project), you probably don't need to do it
|
||||
again.
|
||||
* If you are an individual writing original source code and you're sure you
|
||||
own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual).
|
||||
* If you work for a company that wants to allow you to contribute your work,
|
||||
then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate).
|
||||
|
||||
## Code Reviews
|
||||
Follow either of the two links above to access the appropriate CLA and
|
||||
instructions for how to sign and return it.
|
||||
|
||||
All submissions, including submissions by project members, require review. We
|
||||
use GitHub pull requests for this purpose. Consult
|
||||
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
|
||||
information on using pull requests.
|
||||
## Submitting a Patch
|
||||
|
||||
See [the README](README.md#contributing-to-the-leveldb-project) for areas
|
||||
where we are likely to accept external contributions.
|
||||
1. Sign the contributors license agreement above.
|
||||
2. Decide which code you want to submit. A submission should be a set of changes
|
||||
that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues).
|
||||
Please don't mix more than one logical change per submission, because it makes
|
||||
the history hard to follow. If you want to make a change
|
||||
(e.g. add a sample or feature) that doesn't have a corresponding issue in the
|
||||
issue tracker, please create one.
|
||||
3. **Submitting**: When you are ready to submit, send us a Pull Request. Be
|
||||
sure to include the issue number you fixed and the name you used to sign
|
||||
the CLA.
|
||||
|
||||
## Community Guidelines
|
||||
## Writing Code ##
|
||||
|
||||
This project follows [Google's Open Source Community
|
||||
Guidelines](https://opensource.google/conduct/).
|
||||
If your contribution contains code, please make sure that it follows
|
||||
[the style guide](http://google.github.io/styleguide/cppguide.html).
|
||||
Otherwise we will have to ask you to make changes, and that's no fun for anyone.
|
||||
|
67
README.md
67
README.md
@ -1,11 +1,7 @@
|
||||
LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.
|
||||
**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
|
||||
|
||||
> **This repository is receiving very limited maintenance. We will only review the following types of changes.**
|
||||
>
|
||||
> * Fixes for critical bugs, such as data loss or memory corruption
|
||||
> * Changes absolutely needed by internally supported leveldb clients. These typically fix breakage introduced by a language/standard library/OS update
|
||||
|
||||
[![ci](https://github.com/google/leveldb/actions/workflows/build.yml/badge.svg)](https://github.com/google/leveldb/actions/workflows/build.yml)
|
||||
[![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb)
|
||||
[![Build status](https://ci.appveyor.com/api/projects/status/g2j5j4rfkda6eyw5/branch/master?svg=true)](https://ci.appveyor.com/project/pwnall/leveldb)
|
||||
|
||||
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
|
||||
@ -18,12 +14,12 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
* Multiple changes can be made in one atomic batch.
|
||||
* Users can create a transient snapshot to get a consistent view of data.
|
||||
* Forward and backward iteration is supported over the data.
|
||||
* Data is automatically compressed using the [Snappy compression library](https://google.github.io/snappy/), but [Zstd compression](https://facebook.github.io/zstd/) is also supported.
|
||||
* Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/).
|
||||
* External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
|
||||
|
||||
# Documentation
|
||||
|
||||
[LevelDB library documentation](https://github.com/google/leveldb/blob/main/doc/index.md) is online and bundled with the source code.
|
||||
[LevelDB library documentation](https://github.com/google/leveldb/blob/master/doc/index.md) is online and bundled with the source code.
|
||||
|
||||
# Limitations
|
||||
|
||||
@ -31,12 +27,6 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
* Only a single process (possibly multi-threaded) can access a particular database at a time.
|
||||
* There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library.
|
||||
|
||||
# Getting the Source
|
||||
|
||||
```bash
|
||||
git clone --recurse-submodules https://github.com/google/leveldb.git
|
||||
```
|
||||
|
||||
# Building
|
||||
|
||||
This project supports [CMake](https://cmake.org/) out of the box.
|
||||
@ -77,11 +67,6 @@ Please see the CMake documentation and `CMakeLists.txt` for more advanced usage.
|
||||
|
||||
# Contributing to the leveldb Project
|
||||
|
||||
> **This repository is receiving very limited maintenance. We will only review the following types of changes.**
|
||||
>
|
||||
> * Bug fixes
|
||||
> * Changes absolutely needed by internally supported leveldb clients. These typically fix breakage introduced by a language/standard library/OS update
|
||||
|
||||
The leveldb project welcomes contributions. leveldb's primary goal is to be
|
||||
a reliable and fast key/value store. Changes that are in line with the
|
||||
features/limitations outlined above, and meet the requirements below,
|
||||
@ -101,20 +86,6 @@ Contribution requirements:
|
||||
3. **Tests**: All changes must be accompanied by a new (or changed) test, or
|
||||
a sufficient explanation as to why a new (or changed) test is not required.
|
||||
|
||||
4. **Consistent Style**: This project conforms to the
|
||||
[Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html).
|
||||
To ensure your changes are properly formatted please run:
|
||||
|
||||
```
|
||||
clang-format -i --style=file <file>
|
||||
```
|
||||
|
||||
We are unlikely to accept contributions to the build configuration files, such
|
||||
as `CMakeLists.txt`. We are focused on maintaining a build configuration that
|
||||
allows us to test that the project works in a few supported configurations
|
||||
inside Google. We are not currently interested in supporting other requirements,
|
||||
such as different operating systems, compilers, or build systems.
|
||||
|
||||
## Submitting a Pull Request
|
||||
|
||||
Before any pull request will be accepted the author must first sign a
|
||||
@ -123,7 +94,7 @@ Contributor License Agreement (CLA) at https://cla.developers.google.com/.
|
||||
In order to keep the commit timeline linear
|
||||
[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits)
|
||||
your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase)
|
||||
on google/leveldb/main. This keeps the commit timeline linear and more easily sync'ed
|
||||
on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed
|
||||
with the internal repository at Google. More information at GitHub's
|
||||
[About Git rebase](https://help.github.com/articles/about-git-rebase/) page.
|
||||
|
||||
@ -210,37 +181,37 @@ uncompressed blocks in memory, the read performance improves again:
|
||||
See [doc/index.md](doc/index.md) for more explanation. See
|
||||
[doc/impl.md](doc/impl.md) for a brief overview of the implementation.
|
||||
|
||||
The public interface is in include/leveldb/*.h. Callers should not include or
|
||||
The public interface is in include/*.h. Callers should not include or
|
||||
rely on the details of any other header files in this package. Those
|
||||
internal APIs may be changed without warning.
|
||||
|
||||
Guide to header files:
|
||||
|
||||
* **include/leveldb/db.h**: Main interface to the DB: Start here.
|
||||
* **include/db.h**: Main interface to the DB: Start here
|
||||
|
||||
* **include/leveldb/options.h**: Control over the behavior of an entire database,
|
||||
* **include/options.h**: Control over the behavior of an entire database,
|
||||
and also control over the behavior of individual reads and writes.
|
||||
|
||||
* **include/leveldb/comparator.h**: Abstraction for user-specified comparison function.
|
||||
* **include/comparator.h**: Abstraction for user-specified comparison function.
|
||||
If you want just bytewise comparison of keys, you can use the default
|
||||
comparator, but clients can write their own comparator implementations if they
|
||||
want custom ordering (e.g. to handle different character encodings, etc.).
|
||||
want custom ordering (e.g. to handle different character encodings, etc.)
|
||||
|
||||
* **include/leveldb/iterator.h**: Interface for iterating over data. You can get
|
||||
* **include/iterator.h**: Interface for iterating over data. You can get
|
||||
an iterator from a DB object.
|
||||
|
||||
* **include/leveldb/write_batch.h**: Interface for atomically applying multiple
|
||||
* **include/write_batch.h**: Interface for atomically applying multiple
|
||||
updates to a database.
|
||||
|
||||
* **include/leveldb/slice.h**: A simple module for maintaining a pointer and a
|
||||
* **include/slice.h**: A simple module for maintaining a pointer and a
|
||||
length into some other byte array.
|
||||
|
||||
* **include/leveldb/status.h**: Status is returned from many of the public interfaces
|
||||
* **include/status.h**: Status is returned from many of the public interfaces
|
||||
and is used to report success and various kinds of errors.
|
||||
|
||||
* **include/leveldb/env.h**:
|
||||
* **include/env.h**:
|
||||
Abstraction of the OS environment. A posix implementation of this interface is
|
||||
in util/env_posix.cc.
|
||||
in util/env_posix.cc
|
||||
|
||||
* **include/leveldb/table.h, include/leveldb/table_builder.h**: Lower-level modules that most
|
||||
clients probably won't use directly.
|
||||
* **include/table.h, include/table_builder.h**: Lower-level modules that most
|
||||
clients probably won't use directly
|
||||
|
@ -1,92 +0,0 @@
|
||||
// Copyright (c) 2019 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <cinttypes>
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "benchmark/benchmark.h"
|
||||
#include "db/version_set.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "port/port.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
std::string MakeKey(unsigned int num) {
|
||||
char buf[30];
|
||||
std::snprintf(buf, sizeof(buf), "%016u", num);
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
void BM_LogAndApply(benchmark::State& state) {
|
||||
const int num_base_files = state.range(0);
|
||||
|
||||
std::string dbname = testing::TempDir() + "leveldb_test_benchmark";
|
||||
DestroyDB(dbname, Options());
|
||||
|
||||
DB* db = nullptr;
|
||||
Options opts;
|
||||
opts.create_if_missing = true;
|
||||
Status s = DB::Open(opts, dbname, &db);
|
||||
ASSERT_LEVELDB_OK(s);
|
||||
ASSERT_TRUE(db != nullptr);
|
||||
|
||||
delete db;
|
||||
db = nullptr;
|
||||
|
||||
Env* env = Env::Default();
|
||||
|
||||
port::Mutex mu;
|
||||
MutexLock l(&mu);
|
||||
|
||||
InternalKeyComparator cmp(BytewiseComparator());
|
||||
Options options;
|
||||
VersionSet vset(dbname, &options, nullptr, &cmp);
|
||||
bool save_manifest;
|
||||
ASSERT_LEVELDB_OK(vset.Recover(&save_manifest));
|
||||
VersionEdit vbase;
|
||||
uint64_t fnum = 1;
|
||||
for (int i = 0; i < num_base_files; i++) {
|
||||
InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
|
||||
InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
|
||||
vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
|
||||
}
|
||||
ASSERT_LEVELDB_OK(vset.LogAndApply(&vbase, &mu));
|
||||
|
||||
uint64_t start_micros = env->NowMicros();
|
||||
|
||||
for (auto st : state) {
|
||||
VersionEdit vedit;
|
||||
vedit.RemoveFile(2, fnum);
|
||||
InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
|
||||
InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
|
||||
vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
|
||||
vset.LogAndApply(&vedit, &mu);
|
||||
}
|
||||
|
||||
uint64_t stop_micros = env->NowMicros();
|
||||
unsigned int us = stop_micros - start_micros;
|
||||
char buf[16];
|
||||
std::snprintf(buf, sizeof(buf), "%d", num_base_files);
|
||||
std::fprintf(stderr,
|
||||
"BM_LogAndApply/%-6s %8" PRIu64
|
||||
" iters : %9u us (%7.0f us / iter)\n",
|
||||
buf, state.iterations(), us, ((float)us) / state.iterations());
|
||||
}
|
||||
|
||||
BENCHMARK(BM_LogAndApply)->Arg(1)->Arg(100)->Arg(10000)->Arg(100000);
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
BENCHMARK_MAIN();
|
1
cmake/leveldbConfig.cmake
Normal file
1
cmake/leveldbConfig.cmake
Normal file
@ -0,0 +1 @@
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/leveldbTargets.cmake")
|
@ -1,9 +0,0 @@
|
||||
# Copyright 2019 The LevelDB Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
@PACKAGE_INIT@
|
||||
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/leveldbTargets.cmake")
|
||||
|
||||
check_required_components(leveldb)
|
@ -2,24 +2,29 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class AutoCompactTest : public testing::Test {
|
||||
class AutoCompactTest {
|
||||
public:
|
||||
std::string dbname_;
|
||||
Cache* tiny_cache_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
|
||||
AutoCompactTest() {
|
||||
dbname_ = testing::TempDir() + "autocompact_test";
|
||||
dbname_ = test::TmpDir() + "/autocompact_test";
|
||||
tiny_cache_ = NewLRUCache(100);
|
||||
options_.block_cache = tiny_cache_;
|
||||
DestroyDB(dbname_, options_);
|
||||
options_.create_if_missing = true;
|
||||
options_.compression = kNoCompression;
|
||||
EXPECT_LEVELDB_OK(DB::Open(options_, dbname_, &db_));
|
||||
ASSERT_OK(DB::Open(options_, dbname_, &db_));
|
||||
}
|
||||
|
||||
~AutoCompactTest() {
|
||||
@ -30,7 +35,7 @@ class AutoCompactTest : public testing::Test {
|
||||
|
||||
std::string Key(int i) {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "key%06d", i);
|
||||
snprintf(buf, sizeof(buf), "key%06d", i);
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
@ -42,12 +47,6 @@ class AutoCompactTest : public testing::Test {
|
||||
}
|
||||
|
||||
void DoReads(int n);
|
||||
|
||||
private:
|
||||
std::string dbname_;
|
||||
Cache* tiny_cache_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
};
|
||||
|
||||
static const int kValueSize = 200 * 1024;
|
||||
@ -62,15 +61,15 @@ void AutoCompactTest::DoReads(int n) {
|
||||
|
||||
// Fill database
|
||||
for (int i = 0; i < kCount; i++) {
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), Key(i), value));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
|
||||
}
|
||||
ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable());
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
// Delete everything
|
||||
for (int i = 0; i < kCount; i++) {
|
||||
ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), Key(i)));
|
||||
ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
|
||||
}
|
||||
ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable());
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
// Get initial measurement of the space we will be reading.
|
||||
const int64_t initial_size = Size(Key(0), Key(n));
|
||||
@ -82,16 +81,17 @@ void AutoCompactTest::DoReads(int n) {
|
||||
ASSERT_LT(read, 100) << "Taking too long to compact";
|
||||
Iterator* iter = db_->NewIterator(ReadOptions());
|
||||
for (iter->SeekToFirst();
|
||||
iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) {
|
||||
iter->Valid() && iter->key().ToString() < limit_key;
|
||||
iter->Next()) {
|
||||
// Drop data
|
||||
}
|
||||
delete iter;
|
||||
// Wait a little bit to allow any triggered compactions to complete.
|
||||
Env::Default()->SleepForMicroseconds(1000000);
|
||||
uint64_t size = Size(Key(0), Key(n));
|
||||
std::fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1,
|
||||
size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0);
|
||||
if (size <= initial_size / 10) {
|
||||
fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
|
||||
read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
|
||||
if (size <= initial_size/10) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -100,11 +100,19 @@ void AutoCompactTest::DoReads(int n) {
|
||||
// is pretty much unchanged.
|
||||
const int64_t final_other_size = Size(Key(n), Key(kCount));
|
||||
ASSERT_LE(final_other_size, initial_other_size + 1048576);
|
||||
ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576);
|
||||
ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
|
||||
}
|
||||
|
||||
TEST_F(AutoCompactTest, ReadAll) { DoReads(kCount); }
|
||||
TEST(AutoCompactTest, ReadAll) {
|
||||
DoReads(kCount);
|
||||
}
|
||||
|
||||
TEST_F(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); }
|
||||
TEST(AutoCompactTest, ReadHalf) {
|
||||
DoReads(kCount/2);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -4,8 +4,8 @@
|
||||
|
||||
#include "db/builder.h"
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "leveldb/db.h"
|
||||
@ -14,8 +14,12 @@
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
TableCache* table_cache, Iterator* iter, FileMetaData* meta) {
|
||||
Status BuildTable(const std::string& dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
TableCache* table_cache,
|
||||
Iterator* iter,
|
||||
FileMetaData* meta) {
|
||||
Status s;
|
||||
meta->file_size = 0;
|
||||
iter->SeekToFirst();
|
||||
@ -30,13 +34,10 @@ Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
|
||||
TableBuilder* builder = new TableBuilder(options, file);
|
||||
meta->smallest.DecodeFrom(iter->key());
|
||||
Slice key;
|
||||
for (; iter->Valid(); iter->Next()) {
|
||||
key = iter->key();
|
||||
builder->Add(key, iter->value());
|
||||
}
|
||||
if (!key.empty()) {
|
||||
Slice key = iter->key();
|
||||
meta->largest.DecodeFrom(key);
|
||||
builder->Add(key, iter->value());
|
||||
}
|
||||
|
||||
// Finish and check for builder errors
|
||||
@ -59,7 +60,8 @@ Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
|
||||
if (s.ok()) {
|
||||
// Verify that the table is usable
|
||||
Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number,
|
||||
Iterator* it = table_cache->NewIterator(ReadOptions(),
|
||||
meta->number,
|
||||
meta->file_size);
|
||||
s = it->status();
|
||||
delete it;
|
||||
@ -74,7 +76,7 @@ Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
if (s.ok() && meta->file_size > 0) {
|
||||
// Keep it
|
||||
} else {
|
||||
env->RemoveFile(fname);
|
||||
env->DeleteFile(fname);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
@ -22,8 +22,12 @@ class VersionEdit;
|
||||
// *meta will be filled with metadata about the generated table.
|
||||
// If no data is present in *iter, meta->file_size will be set to
|
||||
// zero, and no Table file will be produced.
|
||||
Status BuildTable(const std::string& dbname, Env* env, const Options& options,
|
||||
TableCache* table_cache, Iterator* iter, FileMetaData* meta);
|
||||
Status BuildTable(const std::string& dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
TableCache* table_cache,
|
||||
Iterator* iter,
|
||||
FileMetaData* meta);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
368
db/c.cc
368
db/c.cc
@ -4,11 +4,7 @@
|
||||
|
||||
#include "leveldb/c.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
@ -46,72 +42,69 @@ using leveldb::WriteOptions;
|
||||
|
||||
extern "C" {
|
||||
|
||||
struct leveldb_t {
|
||||
DB* rep;
|
||||
};
|
||||
struct leveldb_iterator_t {
|
||||
Iterator* rep;
|
||||
};
|
||||
struct leveldb_writebatch_t {
|
||||
WriteBatch rep;
|
||||
};
|
||||
struct leveldb_snapshot_t {
|
||||
const Snapshot* rep;
|
||||
};
|
||||
struct leveldb_readoptions_t {
|
||||
ReadOptions rep;
|
||||
};
|
||||
struct leveldb_writeoptions_t {
|
||||
WriteOptions rep;
|
||||
};
|
||||
struct leveldb_options_t {
|
||||
Options rep;
|
||||
};
|
||||
struct leveldb_cache_t {
|
||||
Cache* rep;
|
||||
};
|
||||
struct leveldb_seqfile_t {
|
||||
SequentialFile* rep;
|
||||
};
|
||||
struct leveldb_randomfile_t {
|
||||
RandomAccessFile* rep;
|
||||
};
|
||||
struct leveldb_writablefile_t {
|
||||
WritableFile* rep;
|
||||
};
|
||||
struct leveldb_logger_t {
|
||||
Logger* rep;
|
||||
};
|
||||
struct leveldb_filelock_t {
|
||||
FileLock* rep;
|
||||
};
|
||||
struct leveldb_t { DB* rep; };
|
||||
struct leveldb_iterator_t { Iterator* rep; };
|
||||
struct leveldb_writebatch_t { WriteBatch rep; };
|
||||
struct leveldb_snapshot_t { const Snapshot* rep; };
|
||||
struct leveldb_readoptions_t { ReadOptions rep; };
|
||||
struct leveldb_writeoptions_t { WriteOptions rep; };
|
||||
struct leveldb_options_t { Options rep; };
|
||||
struct leveldb_cache_t { Cache* rep; };
|
||||
struct leveldb_seqfile_t { SequentialFile* rep; };
|
||||
struct leveldb_randomfile_t { RandomAccessFile* rep; };
|
||||
struct leveldb_writablefile_t { WritableFile* rep; };
|
||||
struct leveldb_logger_t { Logger* rep; };
|
||||
struct leveldb_filelock_t { FileLock* rep; };
|
||||
|
||||
struct leveldb_comparator_t : public Comparator {
|
||||
~leveldb_comparator_t() override { (*destructor_)(state_); }
|
||||
void* state_;
|
||||
void (*destructor_)(void*);
|
||||
int (*compare_)(
|
||||
void*,
|
||||
const char* a, size_t alen,
|
||||
const char* b, size_t blen);
|
||||
const char* (*name_)(void*);
|
||||
|
||||
int Compare(const Slice& a, const Slice& b) const override {
|
||||
virtual ~leveldb_comparator_t() {
|
||||
(*destructor_)(state_);
|
||||
}
|
||||
|
||||
virtual int Compare(const Slice& a, const Slice& b) const {
|
||||
return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
|
||||
}
|
||||
|
||||
const char* Name() const override { return (*name_)(state_); }
|
||||
virtual const char* Name() const {
|
||||
return (*name_)(state_);
|
||||
}
|
||||
|
||||
// No-ops since the C binding does not support key shortening methods.
|
||||
void FindShortestSeparator(std::string*, const Slice&) const override {}
|
||||
void FindShortSuccessor(std::string* key) const override {}
|
||||
|
||||
void* state_;
|
||||
void (*destructor_)(void*);
|
||||
int (*compare_)(void*, const char* a, size_t alen, const char* b,
|
||||
size_t blen);
|
||||
const char* (*name_)(void*);
|
||||
virtual void FindShortestSeparator(std::string*, const Slice&) const { }
|
||||
virtual void FindShortSuccessor(std::string* key) const { }
|
||||
};
|
||||
|
||||
struct leveldb_filterpolicy_t : public FilterPolicy {
|
||||
~leveldb_filterpolicy_t() override { (*destructor_)(state_); }
|
||||
void* state_;
|
||||
void (*destructor_)(void*);
|
||||
const char* (*name_)(void*);
|
||||
char* (*create_)(
|
||||
void*,
|
||||
const char* const* key_array, const size_t* key_length_array,
|
||||
int num_keys,
|
||||
size_t* filter_length);
|
||||
unsigned char (*key_match_)(
|
||||
void*,
|
||||
const char* key, size_t length,
|
||||
const char* filter, size_t filter_length);
|
||||
|
||||
const char* Name() const override { return (*name_)(state_); }
|
||||
virtual ~leveldb_filterpolicy_t() {
|
||||
(*destructor_)(state_);
|
||||
}
|
||||
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
|
||||
virtual const char* Name() const {
|
||||
return (*name_)(state_);
|
||||
}
|
||||
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
std::vector<const char*> key_pointers(n);
|
||||
std::vector<size_t> key_sizes(n);
|
||||
for (int i = 0; i < n; i++) {
|
||||
@ -121,22 +114,13 @@ struct leveldb_filterpolicy_t : public FilterPolicy {
|
||||
size_t len;
|
||||
char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
|
||||
dst->append(filter, len);
|
||||
std::free(filter);
|
||||
free(filter);
|
||||
}
|
||||
|
||||
bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
|
||||
return (*key_match_)(state_, key.data(), key.size(), filter.data(),
|
||||
filter.size());
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
|
||||
return (*key_match_)(state_, key.data(), key.size(),
|
||||
filter.data(), filter.size());
|
||||
}
|
||||
|
||||
void* state_;
|
||||
void (*destructor_)(void*);
|
||||
const char* (*name_)(void*);
|
||||
char* (*create_)(void*, const char* const* key_array,
|
||||
const size_t* key_length_array, int num_keys,
|
||||
size_t* filter_length);
|
||||
uint8_t (*key_match_)(void*, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length);
|
||||
};
|
||||
|
||||
struct leveldb_env_t {
|
||||
@ -152,21 +136,22 @@ static bool SaveError(char** errptr, const Status& s) {
|
||||
*errptr = strdup(s.ToString().c_str());
|
||||
} else {
|
||||
// TODO(sanjay): Merge with existing error?
|
||||
std::free(*errptr);
|
||||
free(*errptr);
|
||||
*errptr = strdup(s.ToString().c_str());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static char* CopyString(const std::string& str) {
|
||||
char* result =
|
||||
reinterpret_cast<char*>(std::malloc(sizeof(char) * str.size()));
|
||||
std::memcpy(result, str.data(), sizeof(char) * str.size());
|
||||
char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
|
||||
memcpy(result, str.data(), sizeof(char) * str.size());
|
||||
return result;
|
||||
}
|
||||
|
||||
leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name,
|
||||
char** errptr) {
|
||||
leveldb_t* leveldb_open(
|
||||
const leveldb_options_t* options,
|
||||
const char* name,
|
||||
char** errptr) {
|
||||
DB* db;
|
||||
if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
|
||||
return nullptr;
|
||||
@ -181,26 +166,39 @@ void leveldb_close(leveldb_t* db) {
|
||||
delete db;
|
||||
}
|
||||
|
||||
void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen, const char* val, size_t vallen,
|
||||
char** errptr) {
|
||||
void leveldb_put(
|
||||
leveldb_t* db,
|
||||
const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen,
|
||||
const char* val, size_t vallen,
|
||||
char** errptr) {
|
||||
SaveError(errptr,
|
||||
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
|
||||
}
|
||||
|
||||
void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen, char** errptr) {
|
||||
void leveldb_delete(
|
||||
leveldb_t* db,
|
||||
const leveldb_writeoptions_t* options,
|
||||
const char* key, size_t keylen,
|
||||
char** errptr) {
|
||||
SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
|
||||
}
|
||||
|
||||
void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options,
|
||||
leveldb_writebatch_t* batch, char** errptr) {
|
||||
|
||||
void leveldb_write(
|
||||
leveldb_t* db,
|
||||
const leveldb_writeoptions_t* options,
|
||||
leveldb_writebatch_t* batch,
|
||||
char** errptr) {
|
||||
SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
|
||||
}
|
||||
|
||||
char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options,
|
||||
const char* key, size_t keylen, size_t* vallen,
|
||||
char** errptr) {
|
||||
char* leveldb_get(
|
||||
leveldb_t* db,
|
||||
const leveldb_readoptions_t* options,
|
||||
const char* key, size_t keylen,
|
||||
size_t* vallen,
|
||||
char** errptr) {
|
||||
char* result = nullptr;
|
||||
std::string tmp;
|
||||
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
|
||||
@ -217,25 +215,30 @@ char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options,
|
||||
}
|
||||
|
||||
leveldb_iterator_t* leveldb_create_iterator(
|
||||
leveldb_t* db, const leveldb_readoptions_t* options) {
|
||||
leveldb_t* db,
|
||||
const leveldb_readoptions_t* options) {
|
||||
leveldb_iterator_t* result = new leveldb_iterator_t;
|
||||
result->rep = db->rep->NewIterator(options->rep);
|
||||
return result;
|
||||
}
|
||||
|
||||
const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) {
|
||||
const leveldb_snapshot_t* leveldb_create_snapshot(
|
||||
leveldb_t* db) {
|
||||
leveldb_snapshot_t* result = new leveldb_snapshot_t;
|
||||
result->rep = db->rep->GetSnapshot();
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_release_snapshot(leveldb_t* db,
|
||||
const leveldb_snapshot_t* snapshot) {
|
||||
void leveldb_release_snapshot(
|
||||
leveldb_t* db,
|
||||
const leveldb_snapshot_t* snapshot) {
|
||||
db->rep->ReleaseSnapshot(snapshot->rep);
|
||||
delete snapshot;
|
||||
}
|
||||
|
||||
char* leveldb_property_value(leveldb_t* db, const char* propname) {
|
||||
char* leveldb_property_value(
|
||||
leveldb_t* db,
|
||||
const char* propname) {
|
||||
std::string tmp;
|
||||
if (db->rep->GetProperty(Slice(propname), &tmp)) {
|
||||
// We use strdup() since we expect human readable output.
|
||||
@ -245,12 +248,12 @@ char* leveldb_property_value(leveldb_t* db, const char* propname) {
|
||||
}
|
||||
}
|
||||
|
||||
void leveldb_approximate_sizes(leveldb_t* db, int num_ranges,
|
||||
const char* const* range_start_key,
|
||||
const size_t* range_start_key_len,
|
||||
const char* const* range_limit_key,
|
||||
const size_t* range_limit_key_len,
|
||||
uint64_t* sizes) {
|
||||
void leveldb_approximate_sizes(
|
||||
leveldb_t* db,
|
||||
int num_ranges,
|
||||
const char* const* range_start_key, const size_t* range_start_key_len,
|
||||
const char* const* range_limit_key, const size_t* range_limit_key_len,
|
||||
uint64_t* sizes) {
|
||||
Range* ranges = new Range[num_ranges];
|
||||
for (int i = 0; i < num_ranges; i++) {
|
||||
ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
|
||||
@ -260,9 +263,10 @@ void leveldb_approximate_sizes(leveldb_t* db, int num_ranges,
|
||||
delete[] ranges;
|
||||
}
|
||||
|
||||
void leveldb_compact_range(leveldb_t* db, const char* start_key,
|
||||
size_t start_key_len, const char* limit_key,
|
||||
size_t limit_key_len) {
|
||||
void leveldb_compact_range(
|
||||
leveldb_t* db,
|
||||
const char* start_key, size_t start_key_len,
|
||||
const char* limit_key, size_t limit_key_len) {
|
||||
Slice a, b;
|
||||
db->rep->CompactRange(
|
||||
// Pass null Slice if corresponding "const char*" is null
|
||||
@ -270,13 +274,17 @@ void leveldb_compact_range(leveldb_t* db, const char* start_key,
|
||||
(limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
|
||||
}
|
||||
|
||||
void leveldb_destroy_db(const leveldb_options_t* options, const char* name,
|
||||
char** errptr) {
|
||||
void leveldb_destroy_db(
|
||||
const leveldb_options_t* options,
|
||||
const char* name,
|
||||
char** errptr) {
|
||||
SaveError(errptr, DestroyDB(name, options->rep));
|
||||
}
|
||||
|
||||
void leveldb_repair_db(const leveldb_options_t* options, const char* name,
|
||||
char** errptr) {
|
||||
void leveldb_repair_db(
|
||||
const leveldb_options_t* options,
|
||||
const char* name,
|
||||
char** errptr) {
|
||||
SaveError(errptr, RepairDB(name, options->rep));
|
||||
}
|
||||
|
||||
@ -285,7 +293,7 @@ void leveldb_iter_destroy(leveldb_iterator_t* iter) {
|
||||
delete iter;
|
||||
}
|
||||
|
||||
uint8_t leveldb_iter_valid(const leveldb_iterator_t* iter) {
|
||||
unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) {
|
||||
return iter->rep->Valid();
|
||||
}
|
||||
|
||||
@ -301,9 +309,13 @@ void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
|
||||
iter->rep->Seek(Slice(k, klen));
|
||||
}
|
||||
|
||||
void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); }
|
||||
void leveldb_iter_next(leveldb_iterator_t* iter) {
|
||||
iter->rep->Next();
|
||||
}
|
||||
|
||||
void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); }
|
||||
void leveldb_iter_prev(leveldb_iterator_t* iter) {
|
||||
iter->rep->Prev();
|
||||
}
|
||||
|
||||
const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
|
||||
Slice s = iter->rep->key();
|
||||
@ -325,34 +337,41 @@ leveldb_writebatch_t* leveldb_writebatch_create() {
|
||||
return new leveldb_writebatch_t;
|
||||
}
|
||||
|
||||
void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; }
|
||||
void leveldb_writebatch_destroy(leveldb_writebatch_t* b) {
|
||||
delete b;
|
||||
}
|
||||
|
||||
void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); }
|
||||
void leveldb_writebatch_clear(leveldb_writebatch_t* b) {
|
||||
b->rep.Clear();
|
||||
}
|
||||
|
||||
void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key,
|
||||
size_t klen, const char* val, size_t vlen) {
|
||||
void leveldb_writebatch_put(
|
||||
leveldb_writebatch_t* b,
|
||||
const char* key, size_t klen,
|
||||
const char* val, size_t vlen) {
|
||||
b->rep.Put(Slice(key, klen), Slice(val, vlen));
|
||||
}
|
||||
|
||||
void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key,
|
||||
size_t klen) {
|
||||
void leveldb_writebatch_delete(
|
||||
leveldb_writebatch_t* b,
|
||||
const char* key, size_t klen) {
|
||||
b->rep.Delete(Slice(key, klen));
|
||||
}
|
||||
|
||||
void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state,
|
||||
void (*put)(void*, const char* k, size_t klen,
|
||||
const char* v, size_t vlen),
|
||||
void (*deleted)(void*, const char* k,
|
||||
size_t klen)) {
|
||||
void leveldb_writebatch_iterate(
|
||||
const leveldb_writebatch_t* b,
|
||||
void* state,
|
||||
void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
|
||||
void (*deleted)(void*, const char* k, size_t klen)) {
|
||||
class H : public WriteBatch::Handler {
|
||||
public:
|
||||
void* state_;
|
||||
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
|
||||
void (*deleted_)(void*, const char* k, size_t klen);
|
||||
void Put(const Slice& key, const Slice& value) override {
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
(*put_)(state_, key.data(), key.size(), value.data(), value.size());
|
||||
}
|
||||
void Delete(const Slice& key) override {
|
||||
virtual void Delete(const Slice& key) {
|
||||
(*deleted_)(state_, key.data(), key.size());
|
||||
}
|
||||
};
|
||||
@ -363,34 +382,43 @@ void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state,
|
||||
b->rep.Iterate(&handler);
|
||||
}
|
||||
|
||||
void leveldb_writebatch_append(leveldb_writebatch_t* destination,
|
||||
const leveldb_writebatch_t* source) {
|
||||
void leveldb_writebatch_append(leveldb_writebatch_t *destination,
|
||||
const leveldb_writebatch_t *source) {
|
||||
destination->rep.Append(source->rep);
|
||||
}
|
||||
|
||||
leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; }
|
||||
leveldb_options_t* leveldb_options_create() {
|
||||
return new leveldb_options_t;
|
||||
}
|
||||
|
||||
void leveldb_options_destroy(leveldb_options_t* options) { delete options; }
|
||||
void leveldb_options_destroy(leveldb_options_t* options) {
|
||||
delete options;
|
||||
}
|
||||
|
||||
void leveldb_options_set_comparator(leveldb_options_t* opt,
|
||||
leveldb_comparator_t* cmp) {
|
||||
void leveldb_options_set_comparator(
|
||||
leveldb_options_t* opt,
|
||||
leveldb_comparator_t* cmp) {
|
||||
opt->rep.comparator = cmp;
|
||||
}
|
||||
|
||||
void leveldb_options_set_filter_policy(leveldb_options_t* opt,
|
||||
leveldb_filterpolicy_t* policy) {
|
||||
void leveldb_options_set_filter_policy(
|
||||
leveldb_options_t* opt,
|
||||
leveldb_filterpolicy_t* policy) {
|
||||
opt->rep.filter_policy = policy;
|
||||
}
|
||||
|
||||
void leveldb_options_set_create_if_missing(leveldb_options_t* opt, uint8_t v) {
|
||||
void leveldb_options_set_create_if_missing(
|
||||
leveldb_options_t* opt, unsigned char v) {
|
||||
opt->rep.create_if_missing = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_error_if_exists(leveldb_options_t* opt, uint8_t v) {
|
||||
void leveldb_options_set_error_if_exists(
|
||||
leveldb_options_t* opt, unsigned char v) {
|
||||
opt->rep.error_if_exists = v;
|
||||
}
|
||||
|
||||
void leveldb_options_set_paranoid_checks(leveldb_options_t* opt, uint8_t v) {
|
||||
void leveldb_options_set_paranoid_checks(
|
||||
leveldb_options_t* opt, unsigned char v) {
|
||||
opt->rep.paranoid_checks = v;
|
||||
}
|
||||
|
||||
@ -431,9 +459,12 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
|
||||
}
|
||||
|
||||
leveldb_comparator_t* leveldb_comparator_create(
|
||||
void* state, void (*destructor)(void*),
|
||||
int (*compare)(void*, const char* a, size_t alen, const char* b,
|
||||
size_t blen),
|
||||
void* state,
|
||||
void (*destructor)(void*),
|
||||
int (*compare)(
|
||||
void*,
|
||||
const char* a, size_t alen,
|
||||
const char* b, size_t blen),
|
||||
const char* (*name)(void*)) {
|
||||
leveldb_comparator_t* result = new leveldb_comparator_t;
|
||||
result->state_ = state;
|
||||
@ -443,15 +474,22 @@ leveldb_comparator_t* leveldb_comparator_create(
|
||||
return result;
|
||||
}
|
||||
|
||||
void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; }
|
||||
void leveldb_comparator_destroy(leveldb_comparator_t* cmp) {
|
||||
delete cmp;
|
||||
}
|
||||
|
||||
leveldb_filterpolicy_t* leveldb_filterpolicy_create(
|
||||
void* state, void (*destructor)(void*),
|
||||
char* (*create_filter)(void*, const char* const* key_array,
|
||||
const size_t* key_length_array, int num_keys,
|
||||
size_t* filter_length),
|
||||
uint8_t (*key_may_match)(void*, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length),
|
||||
void* state,
|
||||
void (*destructor)(void*),
|
||||
char* (*create_filter)(
|
||||
void*,
|
||||
const char* const* key_array, const size_t* key_length_array,
|
||||
int num_keys,
|
||||
size_t* filter_length),
|
||||
unsigned char (*key_may_match)(
|
||||
void*,
|
||||
const char* key, size_t length,
|
||||
const char* filter, size_t filter_length),
|
||||
const char* (*name)(void*)) {
|
||||
leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
|
||||
result->state_ = state;
|
||||
@ -471,8 +509,7 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
|
||||
// they delegate to a NewBloomFilterPolicy() instead of user
|
||||
// supplied C functions.
|
||||
struct Wrapper : public leveldb_filterpolicy_t {
|
||||
static void DoNothing(void*) {}
|
||||
|
||||
const FilterPolicy* rep_;
|
||||
~Wrapper() { delete rep_; }
|
||||
const char* Name() const { return rep_->Name(); }
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
@ -481,8 +518,7 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
|
||||
bool KeyMayMatch(const Slice& key, const Slice& filter) const {
|
||||
return rep_->KeyMayMatch(key, filter);
|
||||
}
|
||||
|
||||
const FilterPolicy* rep_;
|
||||
static void DoNothing(void*) { }
|
||||
};
|
||||
Wrapper* wrapper = new Wrapper;
|
||||
wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
|
||||
@ -495,19 +531,24 @@ leveldb_readoptions_t* leveldb_readoptions_create() {
|
||||
return new leveldb_readoptions_t;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; }
|
||||
void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) {
|
||||
delete opt;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt,
|
||||
uint8_t v) {
|
||||
void leveldb_readoptions_set_verify_checksums(
|
||||
leveldb_readoptions_t* opt,
|
||||
unsigned char v) {
|
||||
opt->rep.verify_checksums = v;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt, uint8_t v) {
|
||||
void leveldb_readoptions_set_fill_cache(
|
||||
leveldb_readoptions_t* opt, unsigned char v) {
|
||||
opt->rep.fill_cache = v;
|
||||
}
|
||||
|
||||
void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt,
|
||||
const leveldb_snapshot_t* snap) {
|
||||
void leveldb_readoptions_set_snapshot(
|
||||
leveldb_readoptions_t* opt,
|
||||
const leveldb_snapshot_t* snap) {
|
||||
opt->rep.snapshot = (snap ? snap->rep : nullptr);
|
||||
}
|
||||
|
||||
@ -515,9 +556,12 @@ leveldb_writeoptions_t* leveldb_writeoptions_create() {
|
||||
return new leveldb_writeoptions_t;
|
||||
}
|
||||
|
||||
void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; }
|
||||
void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) {
|
||||
delete opt;
|
||||
}
|
||||
|
||||
void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt, uint8_t v) {
|
||||
void leveldb_writeoptions_set_sync(
|
||||
leveldb_writeoptions_t* opt, unsigned char v) {
|
||||
opt->rep.sync = v;
|
||||
}
|
||||
|
||||
@ -550,16 +594,22 @@ char* leveldb_env_get_test_directory(leveldb_env_t* env) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
char* buffer = static_cast<char*>(std::malloc(result.size() + 1));
|
||||
std::memcpy(buffer, result.data(), result.size());
|
||||
char* buffer = static_cast<char*>(malloc(result.size() + 1));
|
||||
memcpy(buffer, result.data(), result.size());
|
||||
buffer[result.size()] = '\0';
|
||||
return buffer;
|
||||
}
|
||||
|
||||
void leveldb_free(void* ptr) { std::free(ptr); }
|
||||
void leveldb_free(void* ptr) {
|
||||
free(ptr);
|
||||
}
|
||||
|
||||
int leveldb_major_version() { return kMajorVersion; }
|
||||
int leveldb_major_version() {
|
||||
return kMajorVersion;
|
||||
}
|
||||
|
||||
int leveldb_minor_version() { return kMinorVersion; }
|
||||
int leveldb_minor_version() {
|
||||
return kMinorVersion;
|
||||
}
|
||||
|
||||
} // end extern "C"
|
||||
|
@ -120,7 +120,7 @@ static const char* CmpName(void* arg) {
|
||||
}
|
||||
|
||||
// Custom filter policy
|
||||
static uint8_t fake_filter_result = 1;
|
||||
static unsigned char fake_filter_result = 1;
|
||||
static void FilterDestroy(void* arg) { }
|
||||
static const char* FilterName(void* arg) {
|
||||
return "TestFilter";
|
||||
@ -135,8 +135,10 @@ static char* FilterCreate(
|
||||
memcpy(result, "fake", 4);
|
||||
return result;
|
||||
}
|
||||
uint8_t FilterKeyMatch(void* arg, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length) {
|
||||
unsigned char FilterKeyMatch(
|
||||
void* arg,
|
||||
const char* key, size_t length,
|
||||
const char* filter, size_t filter_length) {
|
||||
CheckCondition(filter_length == 4);
|
||||
CheckCondition(memcmp(filter, "fake", 4) == 0);
|
||||
return fake_filter_result;
|
||||
|
@ -2,42 +2,48 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <sys/types.h>
|
||||
#include "leveldb/db.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include <sys/types.h>
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_format.h"
|
||||
#include "db/version_set.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
static const int kValueSize = 1000;
|
||||
|
||||
class CorruptionTest : public testing::Test {
|
||||
class CorruptionTest {
|
||||
public:
|
||||
CorruptionTest()
|
||||
: db_(nullptr),
|
||||
dbname_("/memenv/corruption_test"),
|
||||
tiny_cache_(NewLRUCache(100)) {
|
||||
test::ErrorEnv env_;
|
||||
std::string dbname_;
|
||||
Cache* tiny_cache_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
|
||||
CorruptionTest() {
|
||||
tiny_cache_ = NewLRUCache(100);
|
||||
options_.env = &env_;
|
||||
options_.block_cache = tiny_cache_;
|
||||
dbname_ = "/memenv/corruption_test";
|
||||
DestroyDB(dbname_, options_);
|
||||
|
||||
db_ = nullptr;
|
||||
options_.create_if_missing = true;
|
||||
Reopen();
|
||||
options_.create_if_missing = false;
|
||||
}
|
||||
|
||||
~CorruptionTest() {
|
||||
delete db_;
|
||||
delete tiny_cache_;
|
||||
delete db_;
|
||||
delete tiny_cache_;
|
||||
}
|
||||
|
||||
Status TryReopen() {
|
||||
@ -46,19 +52,21 @@ class CorruptionTest : public testing::Test {
|
||||
return DB::Open(options_, dbname_, &db_);
|
||||
}
|
||||
|
||||
void Reopen() { ASSERT_LEVELDB_OK(TryReopen()); }
|
||||
void Reopen() {
|
||||
ASSERT_OK(TryReopen());
|
||||
}
|
||||
|
||||
void RepairDB() {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
ASSERT_LEVELDB_OK(::leveldb::RepairDB(dbname_, options_));
|
||||
ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
|
||||
}
|
||||
|
||||
void Build(int n) {
|
||||
std::string key_space, value_space;
|
||||
WriteBatch batch;
|
||||
for (int i = 0; i < n; i++) {
|
||||
// if ((i % 100) == 0) std::fprintf(stderr, "@ %d of %d\n", i, n);
|
||||
//if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
|
||||
Slice key = Key(i, &key_space);
|
||||
batch.Clear();
|
||||
batch.Put(key, Value(i, &value_space));
|
||||
@ -68,7 +76,7 @@ class CorruptionTest : public testing::Test {
|
||||
if (i == n - 1) {
|
||||
options.sync = true;
|
||||
}
|
||||
ASSERT_LEVELDB_OK(db_->Write(options, &batch));
|
||||
ASSERT_OK(db_->Write(options, &batch));
|
||||
}
|
||||
}
|
||||
|
||||
@ -87,7 +95,8 @@ class CorruptionTest : public testing::Test {
|
||||
// Ignore boundary keys.
|
||||
continue;
|
||||
}
|
||||
if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
|
||||
if (!ConsumeDecimalNumber(&in, &key) ||
|
||||
!in.empty() ||
|
||||
key < next_expected) {
|
||||
bad_keys++;
|
||||
continue;
|
||||
@ -102,10 +111,9 @@ class CorruptionTest : public testing::Test {
|
||||
}
|
||||
delete iter;
|
||||
|
||||
std::fprintf(
|
||||
stderr,
|
||||
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
|
||||
min_expected, max_expected, correct, bad_keys, bad_values, missed);
|
||||
fprintf(stderr,
|
||||
"expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
|
||||
min_expected, max_expected, correct, bad_keys, bad_values, missed);
|
||||
ASSERT_LE(min_expected, correct);
|
||||
ASSERT_GE(max_expected, correct);
|
||||
}
|
||||
@ -113,13 +121,14 @@ class CorruptionTest : public testing::Test {
|
||||
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
|
||||
// Pick file to corrupt
|
||||
std::vector<std::string> filenames;
|
||||
ASSERT_LEVELDB_OK(env_.target()->GetChildren(dbname_, &filenames));
|
||||
ASSERT_OK(env_.target()->GetChildren(dbname_, &filenames));
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
std::string fname;
|
||||
int picked_number = -1;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
|
||||
if (ParseFileName(filenames[i], &number, &type) &&
|
||||
type == filetype &&
|
||||
int(number) > picked_number) { // Pick latest file
|
||||
fname = dbname_ + "/" + filenames[i];
|
||||
picked_number = number;
|
||||
@ -128,7 +137,7 @@ class CorruptionTest : public testing::Test {
|
||||
ASSERT_TRUE(!fname.empty()) << filetype;
|
||||
|
||||
uint64_t file_size;
|
||||
ASSERT_LEVELDB_OK(env_.target()->GetFileSize(fname, &file_size));
|
||||
ASSERT_OK(env_.target()->GetFileSize(fname, &file_size));
|
||||
|
||||
if (offset < 0) {
|
||||
// Relative to end of file; make it absolute
|
||||
@ -170,7 +179,7 @@ class CorruptionTest : public testing::Test {
|
||||
// Return the ith key
|
||||
Slice Key(int i, std::string* storage) {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "%016d", i);
|
||||
snprintf(buf, sizeof(buf), "%016d", i);
|
||||
storage->assign(buf, strlen(buf));
|
||||
return Slice(*storage);
|
||||
}
|
||||
@ -180,20 +189,12 @@ class CorruptionTest : public testing::Test {
|
||||
Random r(k);
|
||||
return test::RandomString(&r, kValueSize, storage);
|
||||
}
|
||||
|
||||
test::ErrorEnv env_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
|
||||
private:
|
||||
std::string dbname_;
|
||||
Cache* tiny_cache_;
|
||||
};
|
||||
|
||||
TEST_F(CorruptionTest, Recovery) {
|
||||
TEST(CorruptionTest, Recovery) {
|
||||
Build(100);
|
||||
Check(100, 100);
|
||||
Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
|
||||
Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
|
||||
Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
|
||||
Reopen();
|
||||
|
||||
@ -201,13 +202,13 @@ TEST_F(CorruptionTest, Recovery) {
|
||||
Check(36, 36);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, RecoverWriteError) {
|
||||
TEST(CorruptionTest, RecoverWriteError) {
|
||||
env_.writable_file_error_ = true;
|
||||
Status s = TryReopen();
|
||||
ASSERT_TRUE(!s.ok());
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, NewFileErrorDuringWrite) {
|
||||
TEST(CorruptionTest, NewFileErrorDuringWrite) {
|
||||
// Do enough writing to force minor compaction
|
||||
env_.writable_file_error_ = true;
|
||||
const int num = 3 + (Options().write_buffer_size / kValueSize);
|
||||
@ -224,7 +225,7 @@ TEST_F(CorruptionTest, NewFileErrorDuringWrite) {
|
||||
Reopen();
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, TableFile) {
|
||||
TEST(CorruptionTest, TableFile) {
|
||||
Build(100);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
@ -235,7 +236,7 @@ TEST_F(CorruptionTest, TableFile) {
|
||||
Check(90, 99);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, TableFileRepair) {
|
||||
TEST(CorruptionTest, TableFileRepair) {
|
||||
options_.block_size = 2 * kValueSize; // Limit scope of corruption
|
||||
options_.paranoid_checks = true;
|
||||
Reopen();
|
||||
@ -251,7 +252,7 @@ TEST_F(CorruptionTest, TableFileRepair) {
|
||||
Check(95, 99);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, TableFileIndexData) {
|
||||
TEST(CorruptionTest, TableFileIndexData) {
|
||||
Build(10000); // Enough to build multiple Tables
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
@ -261,36 +262,36 @@ TEST_F(CorruptionTest, TableFileIndexData) {
|
||||
Check(5000, 9999);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, MissingDescriptor) {
|
||||
TEST(CorruptionTest, MissingDescriptor) {
|
||||
Build(1000);
|
||||
RepairDB();
|
||||
Reopen();
|
||||
Check(1000, 1000);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, SequenceNumberRecovery) {
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v1"));
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v2"));
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v3"));
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v4"));
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v5"));
|
||||
TEST(CorruptionTest, SequenceNumberRecovery) {
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
|
||||
RepairDB();
|
||||
Reopen();
|
||||
std::string v;
|
||||
ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_EQ("v5", v);
|
||||
// Write something. If sequence number was not recovered properly,
|
||||
// it will be hidden by an earlier write.
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v6"));
|
||||
ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
|
||||
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_EQ("v6", v);
|
||||
Reopen();
|
||||
ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_EQ("v6", v);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, CorruptedDescriptor) {
|
||||
ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "hello"));
|
||||
TEST(CorruptionTest, CorruptedDescriptor) {
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
dbi->TEST_CompactRange(0, nullptr, nullptr);
|
||||
@ -302,11 +303,11 @@ TEST_F(CorruptionTest, CorruptedDescriptor) {
|
||||
RepairDB();
|
||||
Reopen();
|
||||
std::string v;
|
||||
ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
|
||||
ASSERT_EQ("hello", v);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, CompactionInputError) {
|
||||
TEST(CorruptionTest, CompactionInputError) {
|
||||
Build(10);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
@ -321,7 +322,7 @@ TEST_F(CorruptionTest, CompactionInputError) {
|
||||
Check(10000, 10000);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
TEST(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
options_.paranoid_checks = true;
|
||||
options_.write_buffer_size = 512 << 10;
|
||||
Reopen();
|
||||
@ -342,21 +343,24 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, UnrelatedKeys) {
|
||||
TEST(CorruptionTest, UnrelatedKeys) {
|
||||
Build(10);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_CompactMemTable();
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
|
||||
std::string tmp1, tmp2;
|
||||
ASSERT_LEVELDB_OK(
|
||||
db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
|
||||
ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
|
||||
std::string v;
|
||||
ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
|
||||
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
|
||||
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
|
||||
dbi->TEST_CompactMemTable();
|
||||
ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
|
||||
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
|
||||
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -3,13 +3,9 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
@ -36,7 +32,6 @@
|
||||
// readmissing -- read N missing keys in random order
|
||||
// readhot -- read N times in random order from 1% section of DB
|
||||
// seekrandom -- N random seeks
|
||||
// seekordered -- N ordered seeks
|
||||
// open -- cost of opening a DB
|
||||
// crc32c -- repeated crc32c of 4K of data
|
||||
// Meta operations:
|
||||
@ -61,8 +56,7 @@ static const char* FLAGS_benchmarks =
|
||||
"crc32c,"
|
||||
"snappycomp,"
|
||||
"snappyuncomp,"
|
||||
"zstdcomp,"
|
||||
"zstduncomp,";
|
||||
;
|
||||
|
||||
// Number of key/values to place in database
|
||||
static int FLAGS_num = 1000000;
|
||||
@ -83,9 +77,6 @@ static double FLAGS_compression_ratio = 0.5;
|
||||
// Print histogram of operation timings
|
||||
static bool FLAGS_histogram = false;
|
||||
|
||||
// Count the number of string comparisons performed
|
||||
static bool FLAGS_comparisons = false;
|
||||
|
||||
// Number of bytes to buffer in memtable before compacting
|
||||
// (initialized to default value by "main")
|
||||
static int FLAGS_write_buffer_size = 0;
|
||||
@ -109,9 +100,6 @@ static int FLAGS_open_files = 0;
|
||||
// Negative means use default settings.
|
||||
static int FLAGS_bloom_bits = -1;
|
||||
|
||||
// Common key prefix length.
|
||||
static int FLAGS_key_prefix = 0;
|
||||
|
||||
// If true, do not destroy the existing database. If you set this
|
||||
// flag and also specify a benchmark that wants a fresh database, that
|
||||
// benchmark will fail.
|
||||
@ -120,47 +108,14 @@ static bool FLAGS_use_existing_db = false;
|
||||
// If true, reuse existing log/MANIFEST files when re-opening a database.
|
||||
static bool FLAGS_reuse_logs = false;
|
||||
|
||||
// If true, use compression.
|
||||
static bool FLAGS_compression = true;
|
||||
|
||||
// Use the db with the following name.
|
||||
static const char* FLAGS_db = nullptr;
|
||||
|
||||
// ZSTD compression level to try out
|
||||
static int FLAGS_zstd_compression_level = 1;
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
leveldb::Env* g_env = nullptr;
|
||||
|
||||
class CountComparator : public Comparator {
|
||||
public:
|
||||
CountComparator(const Comparator* wrapped) : wrapped_(wrapped) {}
|
||||
~CountComparator() override {}
|
||||
int Compare(const Slice& a, const Slice& b) const override {
|
||||
count_.fetch_add(1, std::memory_order_relaxed);
|
||||
return wrapped_->Compare(a, b);
|
||||
}
|
||||
const char* Name() const override { return wrapped_->Name(); }
|
||||
void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const override {
|
||||
wrapped_->FindShortestSeparator(start, limit);
|
||||
}
|
||||
|
||||
void FindShortSuccessor(std::string* key) const override {
|
||||
return wrapped_->FindShortSuccessor(key);
|
||||
}
|
||||
|
||||
size_t comparisons() const { return count_.load(std::memory_order_relaxed); }
|
||||
|
||||
void reset() { count_.store(0, std::memory_order_relaxed); }
|
||||
|
||||
private:
|
||||
mutable std::atomic<size_t> count_{0};
|
||||
const Comparator* const wrapped_;
|
||||
};
|
||||
|
||||
// Helper for quickly generating random data.
|
||||
class RandomGenerator {
|
||||
private:
|
||||
@ -193,26 +148,6 @@ class RandomGenerator {
|
||||
}
|
||||
};
|
||||
|
||||
class KeyBuffer {
|
||||
public:
|
||||
KeyBuffer() {
|
||||
assert(FLAGS_key_prefix < sizeof(buffer_));
|
||||
memset(buffer_, 'a', FLAGS_key_prefix);
|
||||
}
|
||||
KeyBuffer& operator=(KeyBuffer& other) = delete;
|
||||
KeyBuffer(KeyBuffer& other) = delete;
|
||||
|
||||
void Set(int k) {
|
||||
std::snprintf(buffer_ + FLAGS_key_prefix,
|
||||
sizeof(buffer_) - FLAGS_key_prefix, "%016d", k);
|
||||
}
|
||||
|
||||
Slice slice() const { return Slice(buffer_, FLAGS_key_prefix + 16); }
|
||||
|
||||
private:
|
||||
char buffer_[1024];
|
||||
};
|
||||
|
||||
#if defined(__linux)
|
||||
static Slice TrimSpace(Slice s) {
|
||||
size_t start = 0;
|
||||
@ -220,7 +155,7 @@ static Slice TrimSpace(Slice s) {
|
||||
start++;
|
||||
}
|
||||
size_t limit = s.size();
|
||||
while (limit > start && isspace(s[limit - 1])) {
|
||||
while (limit > start && isspace(s[limit-1])) {
|
||||
limit--;
|
||||
}
|
||||
return Slice(s.data() + start, limit - start);
|
||||
@ -252,12 +187,14 @@ class Stats {
|
||||
|
||||
void Start() {
|
||||
next_report_ = 100;
|
||||
last_op_finish_ = start_;
|
||||
hist_.Clear();
|
||||
done_ = 0;
|
||||
bytes_ = 0;
|
||||
seconds_ = 0;
|
||||
start_ = g_env->NowMicros();
|
||||
finish_ = start_;
|
||||
message_.clear();
|
||||
start_ = finish_ = last_op_finish_ = g_env->NowMicros();
|
||||
}
|
||||
|
||||
void Merge(const Stats& other) {
|
||||
@ -277,7 +214,9 @@ class Stats {
|
||||
seconds_ = (finish_ - start_) * 1e-6;
|
||||
}
|
||||
|
||||
void AddMessage(Slice msg) { AppendWithSpace(&message_, msg); }
|
||||
void AddMessage(Slice msg) {
|
||||
AppendWithSpace(&message_, msg);
|
||||
}
|
||||
|
||||
void FinishedSingleOp() {
|
||||
if (FLAGS_histogram) {
|
||||
@ -285,34 +224,29 @@ class Stats {
|
||||
double micros = now - last_op_finish_;
|
||||
hist_.Add(micros);
|
||||
if (micros > 20000) {
|
||||
std::fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
|
||||
std::fflush(stderr);
|
||||
fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
|
||||
fflush(stderr);
|
||||
}
|
||||
last_op_finish_ = now;
|
||||
}
|
||||
|
||||
done_++;
|
||||
if (done_ >= next_report_) {
|
||||
if (next_report_ < 1000)
|
||||
next_report_ += 100;
|
||||
else if (next_report_ < 5000)
|
||||
next_report_ += 500;
|
||||
else if (next_report_ < 10000)
|
||||
next_report_ += 1000;
|
||||
else if (next_report_ < 50000)
|
||||
next_report_ += 5000;
|
||||
else if (next_report_ < 100000)
|
||||
next_report_ += 10000;
|
||||
else if (next_report_ < 500000)
|
||||
next_report_ += 50000;
|
||||
else
|
||||
next_report_ += 100000;
|
||||
std::fprintf(stderr, "... finished %d ops%30s\r", done_, "");
|
||||
std::fflush(stderr);
|
||||
if (next_report_ < 1000) next_report_ += 100;
|
||||
else if (next_report_ < 5000) next_report_ += 500;
|
||||
else if (next_report_ < 10000) next_report_ += 1000;
|
||||
else if (next_report_ < 50000) next_report_ += 5000;
|
||||
else if (next_report_ < 100000) next_report_ += 10000;
|
||||
else if (next_report_ < 500000) next_report_ += 50000;
|
||||
else next_report_ += 100000;
|
||||
fprintf(stderr, "... finished %d ops%30s\r", done_, "");
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
|
||||
void AddBytes(int64_t n) { bytes_ += n; }
|
||||
void AddBytes(int64_t n) {
|
||||
bytes_ += n;
|
||||
}
|
||||
|
||||
void Report(const Slice& name) {
|
||||
// Pretend at least one op was done in case we are running a benchmark
|
||||
@ -325,20 +259,21 @@ class Stats {
|
||||
// elapsed times.
|
||||
double elapsed = (finish_ - start_) * 1e-6;
|
||||
char rate[100];
|
||||
std::snprintf(rate, sizeof(rate), "%6.1f MB/s",
|
||||
(bytes_ / 1048576.0) / elapsed);
|
||||
snprintf(rate, sizeof(rate), "%6.1f MB/s",
|
||||
(bytes_ / 1048576.0) / elapsed);
|
||||
extra = rate;
|
||||
}
|
||||
AppendWithSpace(&extra, message_);
|
||||
|
||||
std::fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
|
||||
name.ToString().c_str(), seconds_ * 1e6 / done_,
|
||||
(extra.empty() ? "" : " "), extra.c_str());
|
||||
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
|
||||
name.ToString().c_str(),
|
||||
seconds_ * 1e6 / done_,
|
||||
(extra.empty() ? "" : " "),
|
||||
extra.c_str());
|
||||
if (FLAGS_histogram) {
|
||||
std::fprintf(stdout, "Microseconds per op:\n%s\n",
|
||||
hist_.ToString().c_str());
|
||||
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
|
||||
}
|
||||
std::fflush(stdout);
|
||||
fflush(stdout);
|
||||
}
|
||||
};
|
||||
|
||||
@ -359,70 +294,22 @@ struct SharedState {
|
||||
bool start GUARDED_BY(mu);
|
||||
|
||||
SharedState(int total)
|
||||
: cv(&mu), total(total), num_initialized(0), num_done(0), start(false) {}
|
||||
: cv(&mu), total(total), num_initialized(0), num_done(0), start(false) { }
|
||||
};
|
||||
|
||||
// Per-thread state for concurrent executions of the same benchmark.
|
||||
struct ThreadState {
|
||||
int tid; // 0..n-1 when running in n threads
|
||||
Random rand; // Has different seeds for different threads
|
||||
int tid; // 0..n-1 when running in n threads
|
||||
Random rand; // Has different seeds for different threads
|
||||
Stats stats;
|
||||
SharedState* shared;
|
||||
|
||||
ThreadState(int index, int seed) : tid(index), rand(seed), shared(nullptr) {}
|
||||
ThreadState(int index)
|
||||
: tid(index),
|
||||
rand(1000 + index) {
|
||||
}
|
||||
};
|
||||
|
||||
void Compress(
|
||||
ThreadState* thread, std::string name,
|
||||
std::function<bool(const char*, size_t, std::string*)> compress_func) {
|
||||
RandomGenerator gen;
|
||||
Slice input = gen.Generate(Options().block_size);
|
||||
int64_t bytes = 0;
|
||||
int64_t produced = 0;
|
||||
bool ok = true;
|
||||
std::string compressed;
|
||||
while (ok && bytes < 1024 * 1048576) { // Compress 1G
|
||||
ok = compress_func(input.data(), input.size(), &compressed);
|
||||
produced += compressed.size();
|
||||
bytes += input.size();
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
thread->stats.AddMessage("(" + name + " failure)");
|
||||
} else {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "(output: %.1f%%)",
|
||||
(produced * 100.0) / bytes);
|
||||
thread->stats.AddMessage(buf);
|
||||
thread->stats.AddBytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
void Uncompress(
|
||||
ThreadState* thread, std::string name,
|
||||
std::function<bool(const char*, size_t, std::string*)> compress_func,
|
||||
std::function<bool(const char*, size_t, char*)> uncompress_func) {
|
||||
RandomGenerator gen;
|
||||
Slice input = gen.Generate(Options().block_size);
|
||||
std::string compressed;
|
||||
bool ok = compress_func(input.data(), input.size(), &compressed);
|
||||
int64_t bytes = 0;
|
||||
char* uncompressed = new char[input.size()];
|
||||
while (ok && bytes < 1024 * 1048576) { // Compress 1G
|
||||
ok = uncompress_func(compressed.data(), compressed.size(), uncompressed);
|
||||
bytes += input.size();
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
delete[] uncompressed;
|
||||
|
||||
if (!ok) {
|
||||
thread->stats.AddMessage("(" + name + " failure)");
|
||||
} else {
|
||||
thread->stats.AddBytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
class Benchmark {
|
||||
@ -436,61 +323,55 @@ class Benchmark {
|
||||
WriteOptions write_options_;
|
||||
int reads_;
|
||||
int heap_counter_;
|
||||
CountComparator count_comparator_;
|
||||
int total_thread_count_;
|
||||
|
||||
void PrintHeader() {
|
||||
const int kKeySize = 16 + FLAGS_key_prefix;
|
||||
const int kKeySize = 16;
|
||||
PrintEnvironment();
|
||||
std::fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
|
||||
std::fprintf(
|
||||
stdout, "Values: %d bytes each (%d bytes after compression)\n",
|
||||
FLAGS_value_size,
|
||||
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
|
||||
std::fprintf(stdout, "Entries: %d\n", num_);
|
||||
std::fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
|
||||
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
|
||||
1048576.0));
|
||||
std::fprintf(
|
||||
stdout, "FileSize: %.1f MB (estimated)\n",
|
||||
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) /
|
||||
1048576.0));
|
||||
fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
|
||||
fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
|
||||
FLAGS_value_size,
|
||||
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
|
||||
fprintf(stdout, "Entries: %d\n", num_);
|
||||
fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
|
||||
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
|
||||
/ 1048576.0));
|
||||
fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
|
||||
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
|
||||
/ 1048576.0));
|
||||
PrintWarnings();
|
||||
std::fprintf(stdout, "------------------------------------------------\n");
|
||||
fprintf(stdout, "------------------------------------------------\n");
|
||||
}
|
||||
|
||||
void PrintWarnings() {
|
||||
#if defined(__GNUC__) && !defined(__OPTIMIZE__)
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
|
||||
fprintf(stdout,
|
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
|
||||
);
|
||||
#endif
|
||||
#ifndef NDEBUG
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
|
||||
fprintf(stdout,
|
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
|
||||
#endif
|
||||
|
||||
// See if snappy is working by attempting to compress a compressible string
|
||||
const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
|
||||
std::string compressed;
|
||||
if (!port::Snappy_Compress(text, sizeof(text), &compressed)) {
|
||||
std::fprintf(stdout, "WARNING: Snappy compression is not enabled\n");
|
||||
fprintf(stdout, "WARNING: Snappy compression is not enabled\n");
|
||||
} else if (compressed.size() >= sizeof(text)) {
|
||||
std::fprintf(stdout, "WARNING: Snappy compression is not effective\n");
|
||||
fprintf(stdout, "WARNING: Snappy compression is not effective\n");
|
||||
}
|
||||
}
|
||||
|
||||
void PrintEnvironment() {
|
||||
std::fprintf(stderr, "LevelDB: version %d.%d\n", kMajorVersion,
|
||||
kMinorVersion);
|
||||
fprintf(stderr, "LevelDB: version %d.%d\n",
|
||||
kMajorVersion, kMinorVersion);
|
||||
|
||||
#if defined(__linux)
|
||||
time_t now = time(nullptr);
|
||||
std::fprintf(stderr, "Date: %s",
|
||||
ctime(&now)); // ctime() adds newline
|
||||
fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
|
||||
|
||||
FILE* cpuinfo = std::fopen("/proc/cpuinfo", "r");
|
||||
FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
|
||||
if (cpuinfo != nullptr) {
|
||||
char line[1000];
|
||||
int num_cpus = 0;
|
||||
@ -510,32 +391,30 @@ class Benchmark {
|
||||
cache_size = val.ToString();
|
||||
}
|
||||
}
|
||||
std::fclose(cpuinfo);
|
||||
std::fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
|
||||
std::fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
|
||||
fclose(cpuinfo);
|
||||
fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
|
||||
fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
public:
|
||||
Benchmark()
|
||||
: cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : nullptr),
|
||||
filter_policy_(FLAGS_bloom_bits >= 0
|
||||
? NewBloomFilterPolicy(FLAGS_bloom_bits)
|
||||
: nullptr),
|
||||
db_(nullptr),
|
||||
num_(FLAGS_num),
|
||||
value_size_(FLAGS_value_size),
|
||||
entries_per_batch_(1),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
heap_counter_(0),
|
||||
count_comparator_(BytewiseComparator()),
|
||||
total_thread_count_(0) {
|
||||
: cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : nullptr),
|
||||
filter_policy_(FLAGS_bloom_bits >= 0
|
||||
? NewBloomFilterPolicy(FLAGS_bloom_bits)
|
||||
: nullptr),
|
||||
db_(nullptr),
|
||||
num_(FLAGS_num),
|
||||
value_size_(FLAGS_value_size),
|
||||
entries_per_batch_(1),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
heap_counter_(0) {
|
||||
std::vector<std::string> files;
|
||||
g_env->GetChildren(FLAGS_db, &files);
|
||||
for (size_t i = 0; i < files.size(); i++) {
|
||||
if (Slice(files[i]).starts_with("heap-")) {
|
||||
g_env->RemoveFile(std::string(FLAGS_db) + "/" + files[i]);
|
||||
g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
|
||||
}
|
||||
}
|
||||
if (!FLAGS_use_existing_db) {
|
||||
@ -613,8 +492,6 @@ class Benchmark {
|
||||
method = &Benchmark::ReadMissing;
|
||||
} else if (name == Slice("seekrandom")) {
|
||||
method = &Benchmark::SeekRandom;
|
||||
} else if (name == Slice("seekordered")) {
|
||||
method = &Benchmark::SeekOrdered;
|
||||
} else if (name == Slice("readhot")) {
|
||||
method = &Benchmark::ReadHot;
|
||||
} else if (name == Slice("readrandomsmall")) {
|
||||
@ -635,10 +512,6 @@ class Benchmark {
|
||||
method = &Benchmark::SnappyCompress;
|
||||
} else if (name == Slice("snappyuncomp")) {
|
||||
method = &Benchmark::SnappyUncompress;
|
||||
} else if (name == Slice("zstdcomp")) {
|
||||
method = &Benchmark::ZstdCompress;
|
||||
} else if (name == Slice("zstduncomp")) {
|
||||
method = &Benchmark::ZstdUncompress;
|
||||
} else if (name == Slice("heapprofile")) {
|
||||
HeapProfile();
|
||||
} else if (name == Slice("stats")) {
|
||||
@ -646,16 +519,15 @@ class Benchmark {
|
||||
} else if (name == Slice("sstables")) {
|
||||
PrintStats("leveldb.sstables");
|
||||
} else {
|
||||
if (!name.empty()) { // No error message for empty name
|
||||
std::fprintf(stderr, "unknown benchmark '%s'\n",
|
||||
name.ToString().c_str());
|
||||
if (name != Slice()) { // No error message for empty name
|
||||
fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (fresh_db) {
|
||||
if (FLAGS_use_existing_db) {
|
||||
std::fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
|
||||
name.ToString().c_str());
|
||||
fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
|
||||
name.ToString().c_str());
|
||||
method = nullptr;
|
||||
} else {
|
||||
delete db_;
|
||||
@ -716,11 +588,7 @@ class Benchmark {
|
||||
arg[i].bm = this;
|
||||
arg[i].method = method;
|
||||
arg[i].shared = &shared;
|
||||
++total_thread_count_;
|
||||
// Seed the thread's random state deterministically based upon thread
|
||||
// creation across all benchmarks. This ensures that the seeds are unique
|
||||
// but reproducible when rerunning the same set of benchmarks.
|
||||
arg[i].thread = new ThreadState(i, /*seed=*/1000 + total_thread_count_);
|
||||
arg[i].thread = new ThreadState(i);
|
||||
arg[i].thread->shared = &shared;
|
||||
g_env->StartThread(ThreadBody, &arg[i]);
|
||||
}
|
||||
@ -741,11 +609,6 @@ class Benchmark {
|
||||
arg[0].thread->stats.Merge(arg[i].thread->stats);
|
||||
}
|
||||
arg[0].thread->stats.Report(name);
|
||||
if (FLAGS_comparisons) {
|
||||
fprintf(stdout, "Comparisons: %zu\n", count_comparator_.comparisons());
|
||||
count_comparator_.reset();
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
delete arg[i].thread;
|
||||
@ -766,37 +629,57 @@ class Benchmark {
|
||||
bytes += size;
|
||||
}
|
||||
// Print so result is not dead
|
||||
std::fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc));
|
||||
fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc));
|
||||
|
||||
thread->stats.AddBytes(bytes);
|
||||
thread->stats.AddMessage(label);
|
||||
}
|
||||
|
||||
void SnappyCompress(ThreadState* thread) {
|
||||
Compress(thread, "snappy", &port::Snappy_Compress);
|
||||
RandomGenerator gen;
|
||||
Slice input = gen.Generate(Options().block_size);
|
||||
int64_t bytes = 0;
|
||||
int64_t produced = 0;
|
||||
bool ok = true;
|
||||
std::string compressed;
|
||||
while (ok && bytes < 1024 * 1048576) { // Compress 1G
|
||||
ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
|
||||
produced += compressed.size();
|
||||
bytes += input.size();
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
thread->stats.AddMessage("(snappy failure)");
|
||||
} else {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "(output: %.1f%%)",
|
||||
(produced * 100.0) / bytes);
|
||||
thread->stats.AddMessage(buf);
|
||||
thread->stats.AddBytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
void SnappyUncompress(ThreadState* thread) {
|
||||
Uncompress(thread, "snappy", &port::Snappy_Compress,
|
||||
&port::Snappy_Uncompress);
|
||||
}
|
||||
RandomGenerator gen;
|
||||
Slice input = gen.Generate(Options().block_size);
|
||||
std::string compressed;
|
||||
bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
|
||||
int64_t bytes = 0;
|
||||
char* uncompressed = new char[input.size()];
|
||||
while (ok && bytes < 1024 * 1048576) { // Compress 1G
|
||||
ok = port::Snappy_Uncompress(compressed.data(), compressed.size(),
|
||||
uncompressed);
|
||||
bytes += input.size();
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
delete[] uncompressed;
|
||||
|
||||
void ZstdCompress(ThreadState* thread) {
|
||||
Compress(thread, "zstd",
|
||||
[](const char* input, size_t length, std::string* output) {
|
||||
return port::Zstd_Compress(FLAGS_zstd_compression_level, input,
|
||||
length, output);
|
||||
});
|
||||
}
|
||||
|
||||
void ZstdUncompress(ThreadState* thread) {
|
||||
Uncompress(
|
||||
thread, "zstd",
|
||||
[](const char* input, size_t length, std::string* output) {
|
||||
return port::Zstd_Compress(FLAGS_zstd_compression_level, input,
|
||||
length, output);
|
||||
},
|
||||
&port::Zstd_Uncompress);
|
||||
if (!ok) {
|
||||
thread->stats.AddMessage("(snappy failure)");
|
||||
} else {
|
||||
thread->stats.AddBytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
void Open() {
|
||||
@ -808,18 +691,13 @@ class Benchmark {
|
||||
options.write_buffer_size = FLAGS_write_buffer_size;
|
||||
options.max_file_size = FLAGS_max_file_size;
|
||||
options.block_size = FLAGS_block_size;
|
||||
if (FLAGS_comparisons) {
|
||||
options.comparator = &count_comparator_;
|
||||
}
|
||||
options.max_open_files = FLAGS_open_files;
|
||||
options.filter_policy = filter_policy_;
|
||||
options.reuse_logs = FLAGS_reuse_logs;
|
||||
options.compression =
|
||||
FLAGS_compression ? kSnappyCompression : kNoCompression;
|
||||
Status s = DB::Open(options, FLAGS_db, &db_);
|
||||
if (!s.ok()) {
|
||||
std::fprintf(stderr, "open error: %s\n", s.ToString().c_str());
|
||||
std::exit(1);
|
||||
fprintf(stderr, "open error: %s\n", s.ToString().c_str());
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
@ -831,14 +709,18 @@ class Benchmark {
|
||||
}
|
||||
}
|
||||
|
||||
void WriteSeq(ThreadState* thread) { DoWrite(thread, true); }
|
||||
void WriteSeq(ThreadState* thread) {
|
||||
DoWrite(thread, true);
|
||||
}
|
||||
|
||||
void WriteRandom(ThreadState* thread) { DoWrite(thread, false); }
|
||||
void WriteRandom(ThreadState* thread) {
|
||||
DoWrite(thread, false);
|
||||
}
|
||||
|
||||
void DoWrite(ThreadState* thread, bool seq) {
|
||||
if (num_ != FLAGS_num) {
|
||||
char msg[100];
|
||||
std::snprintf(msg, sizeof(msg), "(%d ops)", num_);
|
||||
snprintf(msg, sizeof(msg), "(%d ops)", num_);
|
||||
thread->stats.AddMessage(msg);
|
||||
}
|
||||
|
||||
@ -846,20 +728,20 @@ class Benchmark {
|
||||
WriteBatch batch;
|
||||
Status s;
|
||||
int64_t bytes = 0;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < num_; i += entries_per_batch_) {
|
||||
batch.Clear();
|
||||
for (int j = 0; j < entries_per_batch_; j++) {
|
||||
const int k = seq ? i + j : thread->rand.Uniform(FLAGS_num);
|
||||
key.Set(k);
|
||||
batch.Put(key.slice(), gen.Generate(value_size_));
|
||||
bytes += value_size_ + key.slice().size();
|
||||
const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
|
||||
char key[100];
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
batch.Put(key, gen.Generate(value_size_));
|
||||
bytes += value_size_ + strlen(key);
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
s = db_->Write(write_options_, &batch);
|
||||
if (!s.ok()) {
|
||||
std::fprintf(stderr, "put error: %s\n", s.ToString().c_str());
|
||||
std::exit(1);
|
||||
fprintf(stderr, "put error: %s\n", s.ToString().c_str());
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
thread->stats.AddBytes(bytes);
|
||||
@ -895,29 +777,28 @@ class Benchmark {
|
||||
ReadOptions options;
|
||||
std::string value;
|
||||
int found = 0;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
const int k = thread->rand.Uniform(FLAGS_num);
|
||||
key.Set(k);
|
||||
if (db_->Get(options, key.slice(), &value).ok()) {
|
||||
char key[100];
|
||||
const int k = thread->rand.Next() % FLAGS_num;
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
if (db_->Get(options, key, &value).ok()) {
|
||||
found++;
|
||||
}
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
char msg[100];
|
||||
std::snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
|
||||
snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
|
||||
thread->stats.AddMessage(msg);
|
||||
}
|
||||
|
||||
void ReadMissing(ThreadState* thread) {
|
||||
ReadOptions options;
|
||||
std::string value;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
const int k = thread->rand.Uniform(FLAGS_num);
|
||||
key.Set(k);
|
||||
Slice s = Slice(key.slice().data(), key.slice().size() - 1);
|
||||
db_->Get(options, s, &value);
|
||||
char key[100];
|
||||
const int k = thread->rand.Next() % FLAGS_num;
|
||||
snprintf(key, sizeof(key), "%016d.", k);
|
||||
db_->Get(options, key, &value);
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
}
|
||||
@ -926,11 +807,11 @@ class Benchmark {
|
||||
ReadOptions options;
|
||||
std::string value;
|
||||
const int range = (FLAGS_num + 99) / 100;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
const int k = thread->rand.Uniform(range);
|
||||
key.Set(k);
|
||||
db_->Get(options, key.slice(), &value);
|
||||
char key[100];
|
||||
const int k = thread->rand.Next() % range;
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
db_->Get(options, key, &value);
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
}
|
||||
@ -938,13 +819,13 @@ class Benchmark {
|
||||
void SeekRandom(ThreadState* thread) {
|
||||
ReadOptions options;
|
||||
int found = 0;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
Iterator* iter = db_->NewIterator(options);
|
||||
const int k = thread->rand.Uniform(FLAGS_num);
|
||||
key.Set(k);
|
||||
iter->Seek(key.slice());
|
||||
if (iter->Valid() && iter->key() == key.slice()) found++;
|
||||
char key[100];
|
||||
const int k = thread->rand.Next() % FLAGS_num;
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
iter->Seek(key);
|
||||
if (iter->Valid() && iter->key() == key) found++;
|
||||
delete iter;
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
@ -953,49 +834,34 @@ class Benchmark {
|
||||
thread->stats.AddMessage(msg);
|
||||
}
|
||||
|
||||
void SeekOrdered(ThreadState* thread) {
|
||||
ReadOptions options;
|
||||
Iterator* iter = db_->NewIterator(options);
|
||||
int found = 0;
|
||||
int k = 0;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
k = (k + (thread->rand.Uniform(100))) % FLAGS_num;
|
||||
key.Set(k);
|
||||
iter->Seek(key.slice());
|
||||
if (iter->Valid() && iter->key() == key.slice()) found++;
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
delete iter;
|
||||
char msg[100];
|
||||
std::snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
|
||||
thread->stats.AddMessage(msg);
|
||||
}
|
||||
|
||||
void DoDelete(ThreadState* thread, bool seq) {
|
||||
RandomGenerator gen;
|
||||
WriteBatch batch;
|
||||
Status s;
|
||||
KeyBuffer key;
|
||||
for (int i = 0; i < num_; i += entries_per_batch_) {
|
||||
batch.Clear();
|
||||
for (int j = 0; j < entries_per_batch_; j++) {
|
||||
const int k = seq ? i + j : (thread->rand.Uniform(FLAGS_num));
|
||||
key.Set(k);
|
||||
batch.Delete(key.slice());
|
||||
const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
|
||||
char key[100];
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
batch.Delete(key);
|
||||
thread->stats.FinishedSingleOp();
|
||||
}
|
||||
s = db_->Write(write_options_, &batch);
|
||||
if (!s.ok()) {
|
||||
std::fprintf(stderr, "del error: %s\n", s.ToString().c_str());
|
||||
std::exit(1);
|
||||
fprintf(stderr, "del error: %s\n", s.ToString().c_str());
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DeleteSeq(ThreadState* thread) { DoDelete(thread, true); }
|
||||
void DeleteSeq(ThreadState* thread) {
|
||||
DoDelete(thread, true);
|
||||
}
|
||||
|
||||
void DeleteRandom(ThreadState* thread) { DoDelete(thread, false); }
|
||||
void DeleteRandom(ThreadState* thread) {
|
||||
DoDelete(thread, false);
|
||||
}
|
||||
|
||||
void ReadWhileWriting(ThreadState* thread) {
|
||||
if (thread->tid > 0) {
|
||||
@ -1003,7 +869,6 @@ class Benchmark {
|
||||
} else {
|
||||
// Special thread that keeps writing until other threads are done.
|
||||
RandomGenerator gen;
|
||||
KeyBuffer key;
|
||||
while (true) {
|
||||
{
|
||||
MutexLock l(&thread->shared->mu);
|
||||
@ -1013,13 +878,13 @@ class Benchmark {
|
||||
}
|
||||
}
|
||||
|
||||
const int k = thread->rand.Uniform(FLAGS_num);
|
||||
key.Set(k);
|
||||
Status s =
|
||||
db_->Put(write_options_, key.slice(), gen.Generate(value_size_));
|
||||
const int k = thread->rand.Next() % FLAGS_num;
|
||||
char key[100];
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
Status s = db_->Put(write_options_, key, gen.Generate(value_size_));
|
||||
if (!s.ok()) {
|
||||
std::fprintf(stderr, "put error: %s\n", s.ToString().c_str());
|
||||
std::exit(1);
|
||||
fprintf(stderr, "put error: %s\n", s.ToString().c_str());
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1028,14 +893,16 @@ class Benchmark {
|
||||
}
|
||||
}
|
||||
|
||||
void Compact(ThreadState* thread) { db_->CompactRange(nullptr, nullptr); }
|
||||
void Compact(ThreadState* thread) {
|
||||
db_->CompactRange(nullptr, nullptr);
|
||||
}
|
||||
|
||||
void PrintStats(const char* key) {
|
||||
std::string stats;
|
||||
if (!db_->GetProperty(key, &stats)) {
|
||||
stats = "(failed)";
|
||||
}
|
||||
std::fprintf(stdout, "\n%s\n", stats.c_str());
|
||||
fprintf(stdout, "\n%s\n", stats.c_str());
|
||||
}
|
||||
|
||||
static void WriteToFile(void* arg, const char* buf, int n) {
|
||||
@ -1044,19 +911,18 @@ class Benchmark {
|
||||
|
||||
void HeapProfile() {
|
||||
char fname[100];
|
||||
std::snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db,
|
||||
++heap_counter_);
|
||||
snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
|
||||
WritableFile* file;
|
||||
Status s = g_env->NewWritableFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
std::fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
return;
|
||||
}
|
||||
bool ok = port::GetHeapProfile(WriteToFile, file);
|
||||
delete file;
|
||||
if (!ok) {
|
||||
std::fprintf(stderr, "heap profiling not supported\n");
|
||||
g_env->RemoveFile(fname);
|
||||
fprintf(stderr, "heap profiling not supported\n");
|
||||
g_env->DeleteFile(fname);
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -1081,18 +947,12 @@ int main(int argc, char** argv) {
|
||||
} else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_histogram = n;
|
||||
} else if (sscanf(argv[i], "--comparisons=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_comparisons = n;
|
||||
} else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_use_existing_db = n;
|
||||
} else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_reuse_logs = n;
|
||||
} else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_compression = n;
|
||||
} else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_num = n;
|
||||
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
|
||||
@ -1107,8 +967,6 @@ int main(int argc, char** argv) {
|
||||
FLAGS_max_file_size = n;
|
||||
} else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_block_size = n;
|
||||
} else if (sscanf(argv[i], "--key_prefix=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_key_prefix = n;
|
||||
} else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_cache_size = n;
|
||||
} else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
|
||||
@ -1118,8 +976,8 @@ int main(int argc, char** argv) {
|
||||
} else if (strncmp(argv[i], "--db=", 5) == 0) {
|
||||
FLAGS_db = argv[i] + 5;
|
||||
} else {
|
||||
std::fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
|
||||
std::exit(1);
|
||||
fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1127,9 +985,9 @@ int main(int argc, char** argv) {
|
||||
|
||||
// Choose a location for the test database if none given with --db=<path>
|
||||
if (FLAGS_db == nullptr) {
|
||||
leveldb::g_env->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
leveldb::g_env->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
}
|
||||
|
||||
leveldb::Benchmark benchmark;
|
320
db/db_impl.cc
320
db/db_impl.cc
@ -4,10 +4,11 @@
|
||||
|
||||
#include "db/db_impl.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@ -41,33 +42,16 @@ const int kNumNonTableCacheFiles = 10;
|
||||
|
||||
// Information kept for every waiting writer
|
||||
struct DBImpl::Writer {
|
||||
explicit Writer(port::Mutex* mu)
|
||||
: batch(nullptr), sync(false), done(false), cv(mu) {}
|
||||
|
||||
Status status;
|
||||
WriteBatch* batch;
|
||||
bool sync;
|
||||
bool done;
|
||||
port::CondVar cv;
|
||||
|
||||
explicit Writer(port::Mutex* mu) : cv(mu) { }
|
||||
};
|
||||
|
||||
struct DBImpl::CompactionState {
|
||||
// Files produced by compaction
|
||||
struct Output {
|
||||
uint64_t number;
|
||||
uint64_t file_size;
|
||||
InternalKey smallest, largest;
|
||||
};
|
||||
|
||||
Output* current_output() { return &outputs[outputs.size() - 1]; }
|
||||
|
||||
explicit CompactionState(Compaction* c)
|
||||
: compaction(c),
|
||||
smallest_snapshot(0),
|
||||
outfile(nullptr),
|
||||
builder(nullptr),
|
||||
total_bytes(0) {}
|
||||
|
||||
Compaction* const compaction;
|
||||
|
||||
// Sequence numbers < smallest_snapshot are not significant since we
|
||||
@ -76,6 +60,12 @@ struct DBImpl::CompactionState {
|
||||
// we can drop all entries for the same key with sequence numbers < S.
|
||||
SequenceNumber smallest_snapshot;
|
||||
|
||||
// Files produced by compaction
|
||||
struct Output {
|
||||
uint64_t number;
|
||||
uint64_t file_size;
|
||||
InternalKey smallest, largest;
|
||||
};
|
||||
std::vector<Output> outputs;
|
||||
|
||||
// State kept for output being generated
|
||||
@ -83,6 +73,15 @@ struct DBImpl::CompactionState {
|
||||
TableBuilder* builder;
|
||||
|
||||
uint64_t total_bytes;
|
||||
|
||||
Output* current_output() { return &outputs[outputs.size()-1]; }
|
||||
|
||||
explicit CompactionState(Compaction* c)
|
||||
: compaction(c),
|
||||
outfile(nullptr),
|
||||
builder(nullptr),
|
||||
total_bytes(0) {
|
||||
}
|
||||
};
|
||||
|
||||
// Fix user-supplied options to be reasonable
|
||||
@ -98,10 +97,10 @@ Options SanitizeOptions(const std::string& dbname,
|
||||
Options result = src;
|
||||
result.comparator = icmp;
|
||||
result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr;
|
||||
ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
|
||||
ClipToRange(&result.write_buffer_size, 64 << 10, 1 << 30);
|
||||
ClipToRange(&result.max_file_size, 1 << 20, 1 << 30);
|
||||
ClipToRange(&result.block_size, 1 << 10, 4 << 20);
|
||||
ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
|
||||
ClipToRange(&result.write_buffer_size, 64<<10, 1<<30);
|
||||
ClipToRange(&result.max_file_size, 1<<20, 1<<30);
|
||||
ClipToRange(&result.block_size, 1<<10, 4<<20);
|
||||
if (result.info_log == nullptr) {
|
||||
// Open a log file in the same directory as the db
|
||||
src.env->CreateDir(dbname); // In case it does not exist
|
||||
@ -196,9 +195,6 @@ Status DBImpl::NewDB() {
|
||||
std::string record;
|
||||
new_db.EncodeTo(&record);
|
||||
s = log.AddRecord(record);
|
||||
if (s.ok()) {
|
||||
s = file->Sync();
|
||||
}
|
||||
if (s.ok()) {
|
||||
s = file->Close();
|
||||
}
|
||||
@ -208,7 +204,7 @@ Status DBImpl::NewDB() {
|
||||
// Make "CURRENT" file that points to the new manifest file.
|
||||
s = SetCurrentFile(env_, dbname_, 1);
|
||||
} else {
|
||||
env_->RemoveFile(manifest);
|
||||
env_->DeleteFile(manifest);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@ -222,7 +218,7 @@ void DBImpl::MaybeIgnoreError(Status* s) const {
|
||||
}
|
||||
}
|
||||
|
||||
void DBImpl::RemoveObsoleteFiles() {
|
||||
void DBImpl::DeleteObsoleteFiles() {
|
||||
mutex_.AssertHeld();
|
||||
|
||||
if (!bg_error_.ok()) {
|
||||
@ -239,9 +235,8 @@ void DBImpl::RemoveObsoleteFiles() {
|
||||
env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
std::vector<std::string> files_to_delete;
|
||||
for (std::string& filename : filenames) {
|
||||
if (ParseFileName(filename, &number, &type)) {
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type)) {
|
||||
bool keep = true;
|
||||
switch (type) {
|
||||
case kLogFile:
|
||||
@ -269,27 +264,19 @@ void DBImpl::RemoveObsoleteFiles() {
|
||||
}
|
||||
|
||||
if (!keep) {
|
||||
files_to_delete.push_back(std::move(filename));
|
||||
if (type == kTableFile) {
|
||||
table_cache_->Evict(number);
|
||||
}
|
||||
Log(options_.info_log, "Delete type=%d #%lld\n", static_cast<int>(type),
|
||||
Log(options_.info_log, "Delete type=%d #%lld\n",
|
||||
static_cast<int>(type),
|
||||
static_cast<unsigned long long>(number));
|
||||
env_->DeleteFile(dbname_ + "/" + filenames[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// While deleting all files unblock other threads. All files being deleted
|
||||
// have unique names which will not collide with newly created files and
|
||||
// are therefore safe to delete while allowing other threads to proceed.
|
||||
mutex_.Unlock();
|
||||
for (const std::string& filename : files_to_delete) {
|
||||
env_->RemoveFile(dbname_ + "/" + filename);
|
||||
}
|
||||
mutex_.Lock();
|
||||
}
|
||||
|
||||
Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
|
||||
Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) {
|
||||
mutex_.AssertHeld();
|
||||
|
||||
// Ignore error from CreateDir since the creation of the DB is
|
||||
@ -304,8 +291,6 @@ Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
|
||||
|
||||
if (!env_->FileExists(CurrentFileName(dbname_))) {
|
||||
if (options_.create_if_missing) {
|
||||
Log(options_.info_log, "Creating DB %s since it was missing.",
|
||||
dbname_.c_str());
|
||||
s = NewDB();
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
@ -316,8 +301,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
|
||||
}
|
||||
} else {
|
||||
if (options_.error_if_exists) {
|
||||
return Status::InvalidArgument(dbname_,
|
||||
"exists (error_if_exists is true)");
|
||||
return Status::InvalidArgument(
|
||||
dbname_, "exists (error_if_exists is true)");
|
||||
}
|
||||
}
|
||||
|
||||
@ -355,8 +340,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
|
||||
}
|
||||
if (!expected.empty()) {
|
||||
char buf[50];
|
||||
std::snprintf(buf, sizeof(buf), "%d missing files; e.g.",
|
||||
static_cast<int>(expected.size()));
|
||||
snprintf(buf, sizeof(buf), "%d missing files; e.g.",
|
||||
static_cast<int>(expected.size()));
|
||||
return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin())));
|
||||
}
|
||||
|
||||
@ -390,10 +375,10 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
|
||||
Logger* info_log;
|
||||
const char* fname;
|
||||
Status* status; // null if options_.paranoid_checks==false
|
||||
void Corruption(size_t bytes, const Status& s) override {
|
||||
virtual void Corruption(size_t bytes, const Status& s) {
|
||||
Log(info_log, "%s%s: dropping %d bytes; %s",
|
||||
(this->status == nullptr ? "(ignoring error) " : ""), fname,
|
||||
static_cast<int>(bytes), s.ToString().c_str());
|
||||
(this->status == nullptr ? "(ignoring error) " : ""),
|
||||
fname, static_cast<int>(bytes), s.ToString().c_str());
|
||||
if (this->status != nullptr && this->status->ok()) *this->status = s;
|
||||
}
|
||||
};
|
||||
@ -419,9 +404,10 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
|
||||
// paranoid_checks==false so that corruptions cause entire commits
|
||||
// to be skipped instead of propagating bad information (like overly
|
||||
// large sequence numbers).
|
||||
log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/);
|
||||
log::Reader reader(file, &reporter, true/*checksum*/,
|
||||
0/*initial_offset*/);
|
||||
Log(options_.info_log, "Recovering log #%llu",
|
||||
(unsigned long long)log_number);
|
||||
(unsigned long long) log_number);
|
||||
|
||||
// Read all the records and add to a memtable
|
||||
std::string scratch;
|
||||
@ -429,10 +415,11 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
|
||||
WriteBatch batch;
|
||||
int compactions = 0;
|
||||
MemTable* mem = nullptr;
|
||||
while (reader.ReadRecord(&record, &scratch) && status.ok()) {
|
||||
while (reader.ReadRecord(&record, &scratch) &&
|
||||
status.ok()) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(record.size(),
|
||||
Status::Corruption("log record too small"));
|
||||
reporter.Corruption(
|
||||
record.size(), Status::Corruption("log record too small"));
|
||||
continue;
|
||||
}
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
@ -446,8 +433,9 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
|
||||
if (!status.ok()) {
|
||||
break;
|
||||
}
|
||||
const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) +
|
||||
WriteBatchInternal::Count(&batch) - 1;
|
||||
const SequenceNumber last_seq =
|
||||
WriteBatchInternal::Sequence(&batch) +
|
||||
WriteBatchInternal::Count(&batch) - 1;
|
||||
if (last_seq > *max_sequence) {
|
||||
*max_sequence = last_seq;
|
||||
}
|
||||
@ -511,7 +499,7 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
|
||||
pending_outputs_.insert(meta.number);
|
||||
Iterator* iter = mem->NewIterator();
|
||||
Log(options_.info_log, "Level-0 table #%llu: started",
|
||||
(unsigned long long)meta.number);
|
||||
(unsigned long long) meta.number);
|
||||
|
||||
Status s;
|
||||
{
|
||||
@ -521,11 +509,13 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
|
||||
}
|
||||
|
||||
Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s",
|
||||
(unsigned long long)meta.number, (unsigned long long)meta.file_size,
|
||||
(unsigned long long) meta.number,
|
||||
(unsigned long long) meta.file_size,
|
||||
s.ToString().c_str());
|
||||
delete iter;
|
||||
pending_outputs_.erase(meta.number);
|
||||
|
||||
|
||||
// Note that if file_size is zero, the file has been deleted and
|
||||
// should not be added to the manifest.
|
||||
int level = 0;
|
||||
@ -535,8 +525,8 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
|
||||
if (base != nullptr) {
|
||||
level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
|
||||
}
|
||||
edit->AddFile(level, meta.number, meta.file_size, meta.smallest,
|
||||
meta.largest);
|
||||
edit->AddFile(level, meta.number, meta.file_size,
|
||||
meta.smallest, meta.largest);
|
||||
}
|
||||
|
||||
CompactionStats stats;
|
||||
@ -573,7 +563,7 @@ void DBImpl::CompactMemTable() {
|
||||
imm_->Unref();
|
||||
imm_ = nullptr;
|
||||
has_imm_.store(false, std::memory_order_release);
|
||||
RemoveObsoleteFiles();
|
||||
DeleteObsoleteFiles();
|
||||
} else {
|
||||
RecordBackgroundError(s);
|
||||
}
|
||||
@ -667,7 +657,8 @@ void DBImpl::MaybeScheduleCompaction() {
|
||||
// DB is being deleted; no more background compactions
|
||||
} else if (!bg_error_.ok()) {
|
||||
// Already got an error; no more changes
|
||||
} else if (imm_ == nullptr && manual_compaction_ == nullptr &&
|
||||
} else if (imm_ == nullptr &&
|
||||
manual_compaction_ == nullptr &&
|
||||
!versions_->NeedsCompaction()) {
|
||||
// No work to be done
|
||||
} else {
|
||||
@ -719,7 +710,8 @@ void DBImpl::BackgroundCompaction() {
|
||||
}
|
||||
Log(options_.info_log,
|
||||
"Manual compaction at level-%d from %s .. %s; will stop at %s\n",
|
||||
m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
|
||||
m->level,
|
||||
(m->begin ? m->begin->DebugString().c_str() : "(begin)"),
|
||||
(m->end ? m->end->DebugString().c_str() : "(end)"),
|
||||
(m->done ? "(end)" : manual_end.DebugString().c_str()));
|
||||
} else {
|
||||
@ -733,18 +725,20 @@ void DBImpl::BackgroundCompaction() {
|
||||
// Move file to next level
|
||||
assert(c->num_input_files(0) == 1);
|
||||
FileMetaData* f = c->input(0, 0);
|
||||
c->edit()->RemoveFile(c->level(), f->number);
|
||||
c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest,
|
||||
f->largest);
|
||||
c->edit()->DeleteFile(c->level(), f->number);
|
||||
c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
|
||||
f->smallest, f->largest);
|
||||
status = versions_->LogAndApply(c->edit(), &mutex_);
|
||||
if (!status.ok()) {
|
||||
RecordBackgroundError(status);
|
||||
}
|
||||
VersionSet::LevelSummaryStorage tmp;
|
||||
Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
|
||||
static_cast<unsigned long long>(f->number), c->level() + 1,
|
||||
static_cast<unsigned long long>(f->number),
|
||||
c->level() + 1,
|
||||
static_cast<unsigned long long>(f->file_size),
|
||||
status.ToString().c_str(), versions_->LevelSummary(&tmp));
|
||||
status.ToString().c_str(),
|
||||
versions_->LevelSummary(&tmp));
|
||||
} else {
|
||||
CompactionState* compact = new CompactionState(c);
|
||||
status = DoCompactionWork(compact);
|
||||
@ -753,7 +747,7 @@ void DBImpl::BackgroundCompaction() {
|
||||
}
|
||||
CleanupCompaction(compact);
|
||||
c->ReleaseInputs();
|
||||
RemoveObsoleteFiles();
|
||||
DeleteObsoleteFiles();
|
||||
}
|
||||
delete c;
|
||||
|
||||
@ -762,7 +756,8 @@ void DBImpl::BackgroundCompaction() {
|
||||
} else if (shutting_down_.load(std::memory_order_acquire)) {
|
||||
// Ignore compaction errors found during shutting down
|
||||
} else {
|
||||
Log(options_.info_log, "Compaction error: %s", status.ToString().c_str());
|
||||
Log(options_.info_log,
|
||||
"Compaction error: %s", status.ToString().c_str());
|
||||
}
|
||||
|
||||
if (is_manual) {
|
||||
@ -857,25 +852,31 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
|
||||
|
||||
if (s.ok() && current_entries > 0) {
|
||||
// Verify that the table is usable
|
||||
Iterator* iter =
|
||||
table_cache_->NewIterator(ReadOptions(), output_number, current_bytes);
|
||||
Iterator* iter = table_cache_->NewIterator(ReadOptions(),
|
||||
output_number,
|
||||
current_bytes);
|
||||
s = iter->status();
|
||||
delete iter;
|
||||
if (s.ok()) {
|
||||
Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes",
|
||||
(unsigned long long)output_number, compact->compaction->level(),
|
||||
(unsigned long long)current_entries,
|
||||
(unsigned long long)current_bytes);
|
||||
Log(options_.info_log,
|
||||
"Generated table #%llu@%d: %lld keys, %lld bytes",
|
||||
(unsigned long long) output_number,
|
||||
compact->compaction->level(),
|
||||
(unsigned long long) current_entries,
|
||||
(unsigned long long) current_bytes);
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
Status DBImpl::InstallCompactionResults(CompactionState* compact) {
|
||||
mutex_.AssertHeld();
|
||||
Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
|
||||
compact->compaction->num_input_files(0), compact->compaction->level(),
|
||||
compact->compaction->num_input_files(1), compact->compaction->level() + 1,
|
||||
Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
|
||||
compact->compaction->num_input_files(0),
|
||||
compact->compaction->level(),
|
||||
compact->compaction->num_input_files(1),
|
||||
compact->compaction->level() + 1,
|
||||
static_cast<long long>(compact->total_bytes));
|
||||
|
||||
// Add compaction outputs
|
||||
@ -883,8 +884,9 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
|
||||
const int level = compact->compaction->level();
|
||||
for (size_t i = 0; i < compact->outputs.size(); i++) {
|
||||
const CompactionState::Output& out = compact->outputs[i];
|
||||
compact->compaction->edit()->AddFile(level + 1, out.number, out.file_size,
|
||||
out.smallest, out.largest);
|
||||
compact->compaction->edit()->AddFile(
|
||||
level + 1,
|
||||
out.number, out.file_size, out.smallest, out.largest);
|
||||
}
|
||||
return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
|
||||
}
|
||||
@ -893,8 +895,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
|
||||
const uint64_t start_micros = env_->NowMicros();
|
||||
int64_t imm_micros = 0; // Micros spent doing imm_ compactions
|
||||
|
||||
Log(options_.info_log, "Compacting %d@%d + %d@%d files",
|
||||
compact->compaction->num_input_files(0), compact->compaction->level(),
|
||||
Log(options_.info_log, "Compacting %d@%d + %d@%d files",
|
||||
compact->compaction->num_input_files(0),
|
||||
compact->compaction->level(),
|
||||
compact->compaction->num_input_files(1),
|
||||
compact->compaction->level() + 1);
|
||||
|
||||
@ -907,18 +910,17 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
|
||||
compact->smallest_snapshot = snapshots_.oldest()->sequence_number();
|
||||
}
|
||||
|
||||
Iterator* input = versions_->MakeInputIterator(compact->compaction);
|
||||
|
||||
// Release mutex while we're actually doing the compaction work
|
||||
mutex_.Unlock();
|
||||
|
||||
Iterator* input = versions_->MakeInputIterator(compact->compaction);
|
||||
input->SeekToFirst();
|
||||
Status status;
|
||||
ParsedInternalKey ikey;
|
||||
std::string current_user_key;
|
||||
bool has_current_user_key = false;
|
||||
SequenceNumber last_sequence_for_key = kMaxSequenceNumber;
|
||||
while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) {
|
||||
for (; input->Valid() && !shutting_down_.load(std::memory_order_acquire); ) {
|
||||
// Prioritize immutable compaction work
|
||||
if (has_imm_.load(std::memory_order_relaxed)) {
|
||||
const uint64_t imm_start = env_->NowMicros();
|
||||
@ -950,8 +952,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
|
||||
last_sequence_for_key = kMaxSequenceNumber;
|
||||
} else {
|
||||
if (!has_current_user_key ||
|
||||
user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) !=
|
||||
0) {
|
||||
user_comparator()->Compare(ikey.user_key,
|
||||
Slice(current_user_key)) != 0) {
|
||||
// First occurrence of this user key
|
||||
current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
|
||||
has_current_user_key = true;
|
||||
@ -960,7 +962,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
|
||||
|
||||
if (last_sequence_for_key <= compact->smallest_snapshot) {
|
||||
// Hidden by an newer entry for same user key
|
||||
drop = true; // (A)
|
||||
drop = true; // (A)
|
||||
} else if (ikey.type == kTypeDeletion &&
|
||||
ikey.sequence <= compact->smallest_snapshot &&
|
||||
compact->compaction->IsBaseLevelForKey(ikey.user_key)) {
|
||||
@ -1046,7 +1048,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
|
||||
RecordBackgroundError(status);
|
||||
}
|
||||
VersionSet::LevelSummaryStorage tmp;
|
||||
Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp));
|
||||
Log(options_.info_log,
|
||||
"compacted to: %s", versions_->LevelSummary(&tmp));
|
||||
return status;
|
||||
}
|
||||
|
||||
@ -1059,7 +1062,7 @@ struct IterState {
|
||||
MemTable* const imm GUARDED_BY(mu);
|
||||
|
||||
IterState(port::Mutex* mutex, MemTable* mem, MemTable* imm, Version* version)
|
||||
: mu(mutex), version(version), mem(mem), imm(imm) {}
|
||||
: mu(mutex), version(version), mem(mem), imm(imm) { }
|
||||
};
|
||||
|
||||
static void CleanupIteratorState(void* arg1, void* arg2) {
|
||||
@ -1112,7 +1115,8 @@ int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
|
||||
return versions_->MaxNextLevelOverlappingBytes();
|
||||
}
|
||||
|
||||
Status DBImpl::Get(const ReadOptions& options, const Slice& key,
|
||||
Status DBImpl::Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value) {
|
||||
Status s;
|
||||
MutexLock l(&mutex_);
|
||||
@ -1163,12 +1167,12 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options) {
|
||||
SequenceNumber latest_snapshot;
|
||||
uint32_t seed;
|
||||
Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed);
|
||||
return NewDBIterator(this, user_comparator(), iter,
|
||||
(options.snapshot != nullptr
|
||||
? static_cast<const SnapshotImpl*>(options.snapshot)
|
||||
->sequence_number()
|
||||
: latest_snapshot),
|
||||
seed);
|
||||
return NewDBIterator(
|
||||
this, user_comparator(), iter,
|
||||
(options.snapshot != nullptr
|
||||
? static_cast<const SnapshotImpl*>(options.snapshot)->sequence_number()
|
||||
: latest_snapshot),
|
||||
seed);
|
||||
}
|
||||
|
||||
void DBImpl::RecordReadSample(Slice key) {
|
||||
@ -1197,9 +1201,9 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
|
||||
return DB::Delete(options, key);
|
||||
}
|
||||
|
||||
Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
|
||||
Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
|
||||
Writer w(&mutex_);
|
||||
w.batch = updates;
|
||||
w.batch = my_batch;
|
||||
w.sync = options.sync;
|
||||
w.done = false;
|
||||
|
||||
@ -1213,13 +1217,13 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
|
||||
}
|
||||
|
||||
// May temporarily unlock and wait.
|
||||
Status status = MakeRoomForWrite(updates == nullptr);
|
||||
Status status = MakeRoomForWrite(my_batch == nullptr);
|
||||
uint64_t last_sequence = versions_->LastSequence();
|
||||
Writer* last_writer = &w;
|
||||
if (status.ok() && updates != nullptr) { // nullptr batch is for compactions
|
||||
WriteBatch* write_batch = BuildBatchGroup(&last_writer);
|
||||
WriteBatchInternal::SetSequence(write_batch, last_sequence + 1);
|
||||
last_sequence += WriteBatchInternal::Count(write_batch);
|
||||
if (status.ok() && my_batch != nullptr) { // nullptr batch is for compactions
|
||||
WriteBatch* updates = BuildBatchGroup(&last_writer);
|
||||
WriteBatchInternal::SetSequence(updates, last_sequence + 1);
|
||||
last_sequence += WriteBatchInternal::Count(updates);
|
||||
|
||||
// Add to log and apply to memtable. We can release the lock
|
||||
// during this phase since &w is currently responsible for logging
|
||||
@ -1227,7 +1231,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
|
||||
// into mem_.
|
||||
{
|
||||
mutex_.Unlock();
|
||||
status = log_->AddRecord(WriteBatchInternal::Contents(write_batch));
|
||||
status = log_->AddRecord(WriteBatchInternal::Contents(updates));
|
||||
bool sync_error = false;
|
||||
if (status.ok() && options.sync) {
|
||||
status = logfile_->Sync();
|
||||
@ -1236,7 +1240,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
|
||||
}
|
||||
}
|
||||
if (status.ok()) {
|
||||
status = WriteBatchInternal::InsertInto(write_batch, mem_);
|
||||
status = WriteBatchInternal::InsertInto(updates, mem_);
|
||||
}
|
||||
mutex_.Lock();
|
||||
if (sync_error) {
|
||||
@ -1246,7 +1250,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
|
||||
RecordBackgroundError(status);
|
||||
}
|
||||
}
|
||||
if (write_batch == tmp_batch_) tmp_batch_->Clear();
|
||||
if (updates == tmp_batch_) tmp_batch_->Clear();
|
||||
|
||||
versions_->SetLastSequence(last_sequence);
|
||||
}
|
||||
@ -1285,8 +1289,8 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
|
||||
// original write is small, limit the growth so we do not slow
|
||||
// down the small write too much.
|
||||
size_t max_size = 1 << 20;
|
||||
if (size <= (128 << 10)) {
|
||||
max_size = size + (128 << 10);
|
||||
if (size <= (128<<10)) {
|
||||
max_size = size + (128<<10);
|
||||
}
|
||||
|
||||
*last_writer = first;
|
||||
@ -1332,8 +1336,9 @@ Status DBImpl::MakeRoomForWrite(bool force) {
|
||||
// Yield previous error
|
||||
s = bg_error_;
|
||||
break;
|
||||
} else if (allow_delay && versions_->NumLevelFiles(0) >=
|
||||
config::kL0_SlowdownWritesTrigger) {
|
||||
} else if (
|
||||
allow_delay &&
|
||||
versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) {
|
||||
// We are getting close to hitting a hard limit on the number of
|
||||
// L0 files. Rather than delaying a single write by several
|
||||
// seconds when we hit the hard limit, start delaying each
|
||||
@ -1368,22 +1373,8 @@ Status DBImpl::MakeRoomForWrite(bool force) {
|
||||
versions_->ReuseFileNumber(new_log_number);
|
||||
break;
|
||||
}
|
||||
|
||||
delete log_;
|
||||
|
||||
s = logfile_->Close();
|
||||
if (!s.ok()) {
|
||||
// We may have lost some data written to the previous log file.
|
||||
// Switch to the new log file anyway, but record as a background
|
||||
// error so we do not attempt any more writes.
|
||||
//
|
||||
// We could perhaps attempt to save the memtable corresponding
|
||||
// to log file and suppress the error if that works, but that
|
||||
// would add more complexity in a critical code path.
|
||||
RecordBackgroundError(s);
|
||||
}
|
||||
delete logfile_;
|
||||
|
||||
logfile_ = lfile;
|
||||
logfile_number_ = new_log_number;
|
||||
log_ = new log::Writer(lfile);
|
||||
@ -1391,7 +1382,7 @@ Status DBImpl::MakeRoomForWrite(bool force) {
|
||||
has_imm_.store(true, std::memory_order_release);
|
||||
mem_ = new MemTable(internal_comparator_);
|
||||
mem_->Ref();
|
||||
force = false; // Do not force another compaction if have room
|
||||
force = false; // Do not force another compaction if have room
|
||||
MaybeScheduleCompaction();
|
||||
}
|
||||
}
|
||||
@ -1415,26 +1406,31 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
|
||||
return false;
|
||||
} else {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "%d",
|
||||
versions_->NumLevelFiles(static_cast<int>(level)));
|
||||
snprintf(buf, sizeof(buf), "%d",
|
||||
versions_->NumLevelFiles(static_cast<int>(level)));
|
||||
*value = buf;
|
||||
return true;
|
||||
}
|
||||
} else if (in == "stats") {
|
||||
char buf[200];
|
||||
std::snprintf(buf, sizeof(buf),
|
||||
" Compactions\n"
|
||||
"Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n"
|
||||
"--------------------------------------------------\n");
|
||||
snprintf(buf, sizeof(buf),
|
||||
" Compactions\n"
|
||||
"Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n"
|
||||
"--------------------------------------------------\n"
|
||||
);
|
||||
value->append(buf);
|
||||
for (int level = 0; level < config::kNumLevels; level++) {
|
||||
int files = versions_->NumLevelFiles(level);
|
||||
if (stats_[level].micros > 0 || files > 0) {
|
||||
std::snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n",
|
||||
level, files, versions_->NumLevelBytes(level) / 1048576.0,
|
||||
stats_[level].micros / 1e6,
|
||||
stats_[level].bytes_read / 1048576.0,
|
||||
stats_[level].bytes_written / 1048576.0);
|
||||
snprintf(
|
||||
buf, sizeof(buf),
|
||||
"%3d %8d %8.0f %9.0f %8.0f %9.0f\n",
|
||||
level,
|
||||
files,
|
||||
versions_->NumLevelBytes(level) / 1048576.0,
|
||||
stats_[level].micros / 1e6,
|
||||
stats_[level].bytes_read / 1048576.0,
|
||||
stats_[level].bytes_written / 1048576.0);
|
||||
value->append(buf);
|
||||
}
|
||||
}
|
||||
@ -1451,8 +1447,8 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
|
||||
total_usage += imm_->ApproximateMemoryUsage();
|
||||
}
|
||||
char buf[50];
|
||||
std::snprintf(buf, sizeof(buf), "%llu",
|
||||
static_cast<unsigned long long>(total_usage));
|
||||
snprintf(buf, sizeof(buf), "%llu",
|
||||
static_cast<unsigned long long>(total_usage));
|
||||
value->append(buf);
|
||||
return true;
|
||||
}
|
||||
@ -1460,11 +1456,16 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) {
|
||||
void DBImpl::GetApproximateSizes(
|
||||
const Range* range, int n,
|
||||
uint64_t* sizes) {
|
||||
// TODO(opt): better implementation
|
||||
MutexLock l(&mutex_);
|
||||
Version* v = versions_->current();
|
||||
v->Ref();
|
||||
Version* v;
|
||||
{
|
||||
MutexLock l(&mutex_);
|
||||
versions_->current()->Ref();
|
||||
v = versions_->current();
|
||||
}
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
// Convert user_key into a corresponding internal key.
|
||||
@ -1475,7 +1476,10 @@ void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) {
|
||||
sizes[i] = (limit >= start ? limit - start : 0);
|
||||
}
|
||||
|
||||
v->Unref();
|
||||
{
|
||||
MutexLock l(&mutex_);
|
||||
v->Unref();
|
||||
}
|
||||
}
|
||||
|
||||
// Default implementations of convenience methods that subclasses of DB
|
||||
@ -1492,9 +1496,10 @@ Status DB::Delete(const WriteOptions& opt, const Slice& key) {
|
||||
return Write(opt, &batch);
|
||||
}
|
||||
|
||||
DB::~DB() = default;
|
||||
DB::~DB() { }
|
||||
|
||||
Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
|
||||
Status DB::Open(const Options& options, const std::string& dbname,
|
||||
DB** dbptr) {
|
||||
*dbptr = nullptr;
|
||||
|
||||
DBImpl* impl = new DBImpl(options, dbname);
|
||||
@ -1524,7 +1529,7 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
|
||||
s = impl->versions_->LogAndApply(&edit, &impl->mutex_);
|
||||
}
|
||||
if (s.ok()) {
|
||||
impl->RemoveObsoleteFiles();
|
||||
impl->DeleteObsoleteFiles();
|
||||
impl->MaybeScheduleCompaction();
|
||||
}
|
||||
impl->mutex_.Unlock();
|
||||
@ -1537,7 +1542,8 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
|
||||
return s;
|
||||
}
|
||||
|
||||
Snapshot::~Snapshot() = default;
|
||||
Snapshot::~Snapshot() {
|
||||
}
|
||||
|
||||
Status DestroyDB(const std::string& dbname, const Options& options) {
|
||||
Env* env = options.env;
|
||||
@ -1557,15 +1563,15 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
if (ParseFileName(filenames[i], &number, &type) &&
|
||||
type != kDBLockFile) { // Lock file will be deleted at end
|
||||
Status del = env->RemoveFile(dbname + "/" + filenames[i]);
|
||||
Status del = env->DeleteFile(dbname + "/" + filenames[i]);
|
||||
if (result.ok() && !del.ok()) {
|
||||
result = del;
|
||||
}
|
||||
}
|
||||
}
|
||||
env->UnlockFile(lock); // Ignore error since state is already gone
|
||||
env->RemoveFile(lockname);
|
||||
env->RemoveDir(dbname); // Ignore error in case dir contains other files
|
||||
env->DeleteFile(lockname);
|
||||
env->DeleteDir(dbname); // Ignore error in case dir contains other files
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
94
db/db_impl.h
94
db/db_impl.h
@ -29,25 +29,21 @@ class VersionSet;
|
||||
class DBImpl : public DB {
|
||||
public:
|
||||
DBImpl(const Options& options, const std::string& dbname);
|
||||
|
||||
DBImpl(const DBImpl&) = delete;
|
||||
DBImpl& operator=(const DBImpl&) = delete;
|
||||
|
||||
~DBImpl() override;
|
||||
virtual ~DBImpl();
|
||||
|
||||
// Implementations of the DB interface
|
||||
Status Put(const WriteOptions&, const Slice& key,
|
||||
const Slice& value) override;
|
||||
Status Delete(const WriteOptions&, const Slice& key) override;
|
||||
Status Write(const WriteOptions& options, WriteBatch* updates) override;
|
||||
Status Get(const ReadOptions& options, const Slice& key,
|
||||
std::string* value) override;
|
||||
Iterator* NewIterator(const ReadOptions&) override;
|
||||
const Snapshot* GetSnapshot() override;
|
||||
void ReleaseSnapshot(const Snapshot* snapshot) override;
|
||||
bool GetProperty(const Slice& property, std::string* value) override;
|
||||
void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override;
|
||||
void CompactRange(const Slice* begin, const Slice* end) override;
|
||||
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
|
||||
virtual Status Delete(const WriteOptions&, const Slice& key);
|
||||
virtual Status Write(const WriteOptions& options, WriteBatch* updates);
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value);
|
||||
virtual Iterator* NewIterator(const ReadOptions&);
|
||||
virtual const Snapshot* GetSnapshot();
|
||||
virtual void ReleaseSnapshot(const Snapshot* snapshot);
|
||||
virtual bool GetProperty(const Slice& property, std::string* value);
|
||||
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end);
|
||||
|
||||
// Extra methods (for testing) that are not in the public DB interface
|
||||
|
||||
@ -76,31 +72,6 @@ class DBImpl : public DB {
|
||||
struct CompactionState;
|
||||
struct Writer;
|
||||
|
||||
// Information for a manual compaction
|
||||
struct ManualCompaction {
|
||||
int level;
|
||||
bool done;
|
||||
const InternalKey* begin; // null means beginning of key range
|
||||
const InternalKey* end; // null means end of key range
|
||||
InternalKey tmp_storage; // Used to keep track of compaction progress
|
||||
};
|
||||
|
||||
// Per level compaction stats. stats_[level] stores the stats for
|
||||
// compactions that produced data for the specified "level".
|
||||
struct CompactionStats {
|
||||
CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {}
|
||||
|
||||
void Add(const CompactionStats& c) {
|
||||
this->micros += c.micros;
|
||||
this->bytes_read += c.bytes_read;
|
||||
this->bytes_written += c.bytes_written;
|
||||
}
|
||||
|
||||
int64_t micros;
|
||||
int64_t bytes_read;
|
||||
int64_t bytes_written;
|
||||
};
|
||||
|
||||
Iterator* NewInternalIterator(const ReadOptions&,
|
||||
SequenceNumber* latest_snapshot,
|
||||
uint32_t* seed);
|
||||
@ -116,7 +87,7 @@ class DBImpl : public DB {
|
||||
void MaybeIgnoreError(Status* s) const;
|
||||
|
||||
// Delete any unneeded files and stale in-memory entries.
|
||||
void RemoveObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
void DeleteObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
// Compact the in-memory write buffer to disk. Switches to a new
|
||||
// log-file/memtable and writes a new descriptor iff successful.
|
||||
@ -151,10 +122,6 @@ class DBImpl : public DB {
|
||||
Status InstallCompactionResults(CompactionState* compact)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
|
||||
|
||||
const Comparator* user_comparator() const {
|
||||
return internal_comparator_.user_comparator();
|
||||
}
|
||||
|
||||
// Constant after construction
|
||||
Env* const env_;
|
||||
const InternalKeyComparator internal_comparator_;
|
||||
@ -195,14 +162,45 @@ class DBImpl : public DB {
|
||||
// Has a background compaction been scheduled or is running?
|
||||
bool background_compaction_scheduled_ GUARDED_BY(mutex_);
|
||||
|
||||
// Information for a manual compaction
|
||||
struct ManualCompaction {
|
||||
int level;
|
||||
bool done;
|
||||
const InternalKey* begin; // null means beginning of key range
|
||||
const InternalKey* end; // null means end of key range
|
||||
InternalKey tmp_storage; // Used to keep track of compaction progress
|
||||
};
|
||||
ManualCompaction* manual_compaction_ GUARDED_BY(mutex_);
|
||||
|
||||
VersionSet* const versions_ GUARDED_BY(mutex_);
|
||||
VersionSet* const versions_;
|
||||
|
||||
// Have we encountered a background error in paranoid mode?
|
||||
Status bg_error_ GUARDED_BY(mutex_);
|
||||
|
||||
// Per level compaction stats. stats_[level] stores the stats for
|
||||
// compactions that produced data for the specified "level".
|
||||
struct CompactionStats {
|
||||
int64_t micros;
|
||||
int64_t bytes_read;
|
||||
int64_t bytes_written;
|
||||
|
||||
CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { }
|
||||
|
||||
void Add(const CompactionStats& c) {
|
||||
this->micros += c.micros;
|
||||
this->bytes_read += c.bytes_read;
|
||||
this->bytes_written += c.bytes_written;
|
||||
}
|
||||
};
|
||||
CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_);
|
||||
|
||||
// No copying allowed
|
||||
DBImpl(const DBImpl&);
|
||||
void operator=(const DBImpl&);
|
||||
|
||||
const Comparator* user_comparator() const {
|
||||
return internal_comparator_.user_comparator();
|
||||
}
|
||||
};
|
||||
|
||||
// Sanitize db options. The caller should delete result.info_log if
|
||||
|
@ -4,9 +4,9 @@
|
||||
|
||||
#include "db/db_iter.h"
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "port/port.h"
|
||||
@ -21,9 +21,9 @@ static void DumpInternalIter(Iterator* iter) {
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ParsedInternalKey k;
|
||||
if (!ParseInternalKey(iter->key(), &k)) {
|
||||
std::fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str());
|
||||
fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str());
|
||||
} else {
|
||||
std::fprintf(stderr, "@ '%s'\n", k.DebugString().c_str());
|
||||
fprintf(stderr, "@ '%s'\n", k.DebugString().c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -36,14 +36,17 @@ namespace {
|
||||
// combines multiple entries for the same userkey found in the DB
|
||||
// representation into a single entry while accounting for sequence
|
||||
// numbers, deletion markers, overwrites, etc.
|
||||
class DBIter : public Iterator {
|
||||
class DBIter: public Iterator {
|
||||
public:
|
||||
// Which direction is the iterator currently moving?
|
||||
// (1) When moving forward, the internal iterator is positioned at
|
||||
// the exact entry that yields this->key(), this->value()
|
||||
// (2) When moving backwards, the internal iterator is positioned
|
||||
// just before all entries whose user key == this->key().
|
||||
enum Direction { kForward, kReverse };
|
||||
enum Direction {
|
||||
kForward,
|
||||
kReverse
|
||||
};
|
||||
|
||||
DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
|
||||
uint32_t seed)
|
||||
@ -54,22 +57,21 @@ class DBIter : public Iterator {
|
||||
direction_(kForward),
|
||||
valid_(false),
|
||||
rnd_(seed),
|
||||
bytes_until_read_sampling_(RandomCompactionPeriod()) {}
|
||||
|
||||
DBIter(const DBIter&) = delete;
|
||||
DBIter& operator=(const DBIter&) = delete;
|
||||
|
||||
~DBIter() override { delete iter_; }
|
||||
bool Valid() const override { return valid_; }
|
||||
Slice key() const override {
|
||||
bytes_until_read_sampling_(RandomCompactionPeriod()) {
|
||||
}
|
||||
virtual ~DBIter() {
|
||||
delete iter_;
|
||||
}
|
||||
virtual bool Valid() const { return valid_; }
|
||||
virtual Slice key() const {
|
||||
assert(valid_);
|
||||
return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
|
||||
}
|
||||
Slice value() const override {
|
||||
virtual Slice value() const {
|
||||
assert(valid_);
|
||||
return (direction_ == kForward) ? iter_->value() : saved_value_;
|
||||
}
|
||||
Status status() const override {
|
||||
virtual Status status() const {
|
||||
if (status_.ok()) {
|
||||
return iter_->status();
|
||||
} else {
|
||||
@ -77,11 +79,11 @@ class DBIter : public Iterator {
|
||||
}
|
||||
}
|
||||
|
||||
void Next() override;
|
||||
void Prev() override;
|
||||
void Seek(const Slice& target) override;
|
||||
void SeekToFirst() override;
|
||||
void SeekToLast() override;
|
||||
virtual void Next();
|
||||
virtual void Prev();
|
||||
virtual void Seek(const Slice& target);
|
||||
virtual void SeekToFirst();
|
||||
virtual void SeekToLast();
|
||||
|
||||
private:
|
||||
void FindNextUserEntry(bool skipping, std::string* skip);
|
||||
@ -103,20 +105,26 @@ class DBIter : public Iterator {
|
||||
|
||||
// Picks the number of bytes that can be read until a compaction is scheduled.
|
||||
size_t RandomCompactionPeriod() {
|
||||
return rnd_.Uniform(2 * config::kReadBytesPeriod);
|
||||
return rnd_.Uniform(2*config::kReadBytesPeriod);
|
||||
}
|
||||
|
||||
DBImpl* db_;
|
||||
const Comparator* const user_comparator_;
|
||||
Iterator* const iter_;
|
||||
SequenceNumber const sequence_;
|
||||
|
||||
Status status_;
|
||||
std::string saved_key_; // == current key when direction_==kReverse
|
||||
std::string saved_value_; // == current raw value when direction_==kReverse
|
||||
std::string saved_key_; // == current key when direction_==kReverse
|
||||
std::string saved_value_; // == current raw value when direction_==kReverse
|
||||
Direction direction_;
|
||||
bool valid_;
|
||||
|
||||
Random rnd_;
|
||||
size_t bytes_until_read_sampling_;
|
||||
|
||||
// No copying allowed
|
||||
DBIter(const DBIter&);
|
||||
void operator=(const DBIter&);
|
||||
};
|
||||
|
||||
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
|
||||
@ -160,15 +168,6 @@ void DBIter::Next() {
|
||||
} else {
|
||||
// Store in saved_key_ the current key so we skip it below.
|
||||
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
|
||||
|
||||
// iter_ is pointing to current key. We can now safely move to the next to
|
||||
// avoid checking current key.
|
||||
iter_->Next();
|
||||
if (!iter_->Valid()) {
|
||||
valid_ = false;
|
||||
saved_key_.clear();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
FindNextUserEntry(true, &saved_key_);
|
||||
@ -222,8 +221,8 @@ void DBIter::Prev() {
|
||||
ClearSavedValue();
|
||||
return;
|
||||
}
|
||||
if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) <
|
||||
0) {
|
||||
if (user_comparator_->Compare(ExtractUserKey(iter_->key()),
|
||||
saved_key_) < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -279,8 +278,8 @@ void DBIter::Seek(const Slice& target) {
|
||||
direction_ = kForward;
|
||||
ClearSavedValue();
|
||||
saved_key_.clear();
|
||||
AppendInternalKey(&saved_key_,
|
||||
ParsedInternalKey(target, sequence_, kValueTypeForSeek));
|
||||
AppendInternalKey(
|
||||
&saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
|
||||
iter_->Seek(saved_key_);
|
||||
if (iter_->Valid()) {
|
||||
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
|
||||
@ -309,9 +308,12 @@ void DBIter::SeekToLast() {
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter, SequenceNumber sequence,
|
||||
uint32_t seed) {
|
||||
Iterator* NewDBIterator(
|
||||
DBImpl* db,
|
||||
const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter,
|
||||
SequenceNumber sequence,
|
||||
uint32_t seed) {
|
||||
return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
|
||||
}
|
||||
|
||||
|
11
db/db_iter.h
11
db/db_iter.h
@ -5,10 +5,9 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
#define STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include <stdint.h>
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@ -17,8 +16,10 @@ class DBImpl;
|
||||
// Return a new iterator that converts internal keys (yielded by
|
||||
// "*internal_iter") that were live at the specified "sequence" number
|
||||
// into appropriate user keys.
|
||||
Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter, SequenceNumber sequence,
|
||||
Iterator* NewDBIterator(DBImpl* db,
|
||||
const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter,
|
||||
SequenceNumber sequence,
|
||||
uint32_t seed);
|
||||
|
||||
} // namespace leveldb
|
||||
|
956
db/db_test.cc
956
db/db_test.cc
File diff suppressed because it is too large
Load Diff
@ -2,11 +2,8 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
#include "db/dbformat.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <sstream>
|
||||
|
||||
#include "port/port.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
@ -24,20 +21,26 @@ void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
|
||||
}
|
||||
|
||||
std::string ParsedInternalKey::DebugString() const {
|
||||
std::ostringstream ss;
|
||||
ss << '\'' << EscapeString(user_key.ToString()) << "' @ " << sequence << " : "
|
||||
<< static_cast<int>(type);
|
||||
return ss.str();
|
||||
char buf[50];
|
||||
snprintf(buf, sizeof(buf), "' @ %llu : %d",
|
||||
(unsigned long long) sequence,
|
||||
int(type));
|
||||
std::string result = "'";
|
||||
result += EscapeString(user_key.ToString());
|
||||
result += buf;
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string InternalKey::DebugString() const {
|
||||
std::string result;
|
||||
ParsedInternalKey parsed;
|
||||
if (ParseInternalKey(rep_, &parsed)) {
|
||||
return parsed.DebugString();
|
||||
result = parsed.DebugString();
|
||||
} else {
|
||||
result = "(bad)";
|
||||
result.append(EscapeString(rep_));
|
||||
}
|
||||
std::ostringstream ss;
|
||||
ss << "(bad)" << EscapeString(rep_);
|
||||
return ss.str();
|
||||
return result;
|
||||
}
|
||||
|
||||
const char* InternalKeyComparator::Name() const {
|
||||
@ -62,8 +65,9 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
|
||||
return r;
|
||||
}
|
||||
|
||||
void InternalKeyComparator::FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const {
|
||||
void InternalKeyComparator::FindShortestSeparator(
|
||||
std::string* start,
|
||||
const Slice& limit) const {
|
||||
// Attempt to shorten the user portion of the key
|
||||
Slice user_start = ExtractUserKey(*start);
|
||||
Slice user_limit = ExtractUserKey(limit);
|
||||
@ -73,8 +77,7 @@ void InternalKeyComparator::FindShortestSeparator(std::string* start,
|
||||
user_comparator_->Compare(user_start, tmp) < 0) {
|
||||
// User key has become shorter physically, but larger logically.
|
||||
// Tack on the earliest possible number to the shortened user key.
|
||||
PutFixed64(&tmp,
|
||||
PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
|
||||
PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
|
||||
assert(this->Compare(*start, tmp) < 0);
|
||||
assert(this->Compare(tmp, limit) < 0);
|
||||
start->swap(tmp);
|
||||
@ -89,14 +92,15 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
|
||||
user_comparator_->Compare(user_key, tmp) < 0) {
|
||||
// User key has become shorter physically, but larger logically.
|
||||
// Tack on the earliest possible number to the shortened user key.
|
||||
PutFixed64(&tmp,
|
||||
PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
|
||||
PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
|
||||
assert(this->Compare(*key, tmp) < 0);
|
||||
key->swap(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); }
|
||||
const char* InternalFilterPolicy::Name() const {
|
||||
return user_policy_->Name();
|
||||
}
|
||||
|
||||
void InternalFilterPolicy::CreateFilter(const Slice* keys, int n,
|
||||
std::string* dst) const {
|
||||
@ -126,7 +130,7 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
|
||||
start_ = dst;
|
||||
dst = EncodeVarint32(dst, usize + 8);
|
||||
kstart_ = dst;
|
||||
std::memcpy(dst, user_key.data(), usize);
|
||||
memcpy(dst, user_key.data(), usize);
|
||||
dst += usize;
|
||||
EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
|
||||
dst += 8;
|
||||
|
@ -5,10 +5,7 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include <stdio.h>
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
@ -51,7 +48,10 @@ class InternalKey;
|
||||
// Value types encoded as the last component of internal keys.
|
||||
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
|
||||
// data structures.
|
||||
enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 };
|
||||
enum ValueType {
|
||||
kTypeDeletion = 0x0,
|
||||
kTypeValue = 0x1
|
||||
};
|
||||
// kValueTypeForSeek defines the ValueType that should be passed when
|
||||
// constructing a ParsedInternalKey object for seeking to a particular
|
||||
// sequence number (since we sort sequence numbers in decreasing order
|
||||
@ -64,16 +64,17 @@ typedef uint64_t SequenceNumber;
|
||||
|
||||
// We leave eight bits empty at the bottom so a type and sequence#
|
||||
// can be packed together into 64-bits.
|
||||
static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1);
|
||||
static const SequenceNumber kMaxSequenceNumber =
|
||||
((0x1ull << 56) - 1);
|
||||
|
||||
struct ParsedInternalKey {
|
||||
Slice user_key;
|
||||
SequenceNumber sequence;
|
||||
ValueType type;
|
||||
|
||||
ParsedInternalKey() {} // Intentionally left uninitialized (for speed)
|
||||
ParsedInternalKey() { } // Intentionally left uninitialized (for speed)
|
||||
ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
|
||||
: user_key(u), sequence(seq), type(t) {}
|
||||
: user_key(u), sequence(seq), type(t) { }
|
||||
std::string DebugString() const;
|
||||
};
|
||||
|
||||
@ -102,14 +103,14 @@ inline Slice ExtractUserKey(const Slice& internal_key) {
|
||||
class InternalKeyComparator : public Comparator {
|
||||
private:
|
||||
const Comparator* user_comparator_;
|
||||
|
||||
public:
|
||||
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {}
|
||||
const char* Name() const override;
|
||||
int Compare(const Slice& a, const Slice& b) const override;
|
||||
void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const override;
|
||||
void FindShortSuccessor(std::string* key) const override;
|
||||
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { }
|
||||
virtual const char* Name() const;
|
||||
virtual int Compare(const Slice& a, const Slice& b) const;
|
||||
virtual void FindShortestSeparator(
|
||||
std::string* start,
|
||||
const Slice& limit) const;
|
||||
virtual void FindShortSuccessor(std::string* key) const;
|
||||
|
||||
const Comparator* user_comparator() const { return user_comparator_; }
|
||||
|
||||
@ -120,12 +121,11 @@ class InternalKeyComparator : public Comparator {
|
||||
class InternalFilterPolicy : public FilterPolicy {
|
||||
private:
|
||||
const FilterPolicy* const user_policy_;
|
||||
|
||||
public:
|
||||
explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {}
|
||||
const char* Name() const override;
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const override;
|
||||
bool KeyMayMatch(const Slice& key, const Slice& filter) const override;
|
||||
explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { }
|
||||
virtual const char* Name() const;
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const;
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
|
||||
};
|
||||
|
||||
// Modules in this directory should keep internal keys wrapped inside
|
||||
@ -134,18 +134,13 @@ class InternalFilterPolicy : public FilterPolicy {
|
||||
class InternalKey {
|
||||
private:
|
||||
std::string rep_;
|
||||
|
||||
public:
|
||||
InternalKey() {} // Leave rep_ as empty to indicate it is invalid
|
||||
InternalKey() { } // Leave rep_ as empty to indicate it is invalid
|
||||
InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
|
||||
AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
|
||||
}
|
||||
|
||||
bool DecodeFrom(const Slice& s) {
|
||||
rep_.assign(s.data(), s.size());
|
||||
return !rep_.empty();
|
||||
}
|
||||
|
||||
void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
|
||||
Slice Encode() const {
|
||||
assert(!rep_.empty());
|
||||
return rep_;
|
||||
@ -163,8 +158,8 @@ class InternalKey {
|
||||
std::string DebugString() const;
|
||||
};
|
||||
|
||||
inline int InternalKeyComparator::Compare(const InternalKey& a,
|
||||
const InternalKey& b) const {
|
||||
inline int InternalKeyComparator::Compare(
|
||||
const InternalKey& a, const InternalKey& b) const {
|
||||
return Compare(a.Encode(), b.Encode());
|
||||
}
|
||||
|
||||
@ -173,11 +168,11 @@ inline bool ParseInternalKey(const Slice& internal_key,
|
||||
const size_t n = internal_key.size();
|
||||
if (n < 8) return false;
|
||||
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
||||
uint8_t c = num & 0xff;
|
||||
unsigned char c = num & 0xff;
|
||||
result->sequence = num >> 8;
|
||||
result->type = static_cast<ValueType>(c);
|
||||
result->user_key = Slice(internal_key.data(), n - 8);
|
||||
return (c <= static_cast<uint8_t>(kTypeValue));
|
||||
return (c <= static_cast<unsigned char>(kTypeValue));
|
||||
}
|
||||
|
||||
// A helper class useful for DBImpl::Get()
|
||||
@ -187,9 +182,6 @@ class LookupKey {
|
||||
// the specified sequence number.
|
||||
LookupKey(const Slice& user_key, SequenceNumber sequence);
|
||||
|
||||
LookupKey(const LookupKey&) = delete;
|
||||
LookupKey& operator=(const LookupKey&) = delete;
|
||||
|
||||
~LookupKey();
|
||||
|
||||
// Return a key suitable for lookup in a MemTable.
|
||||
@ -212,7 +204,11 @@ class LookupKey {
|
||||
const char* start_;
|
||||
const char* kstart_;
|
||||
const char* end_;
|
||||
char space_[200]; // Avoid allocation for short keys
|
||||
char space_[200]; // Avoid allocation for short keys
|
||||
|
||||
// No copying allowed
|
||||
LookupKey(const LookupKey&);
|
||||
void operator=(const LookupKey&);
|
||||
};
|
||||
|
||||
inline LookupKey::~LookupKey() {
|
||||
|
@ -3,13 +3,13 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/dbformat.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
static std::string IKey(const std::string& user_key, uint64_t seq,
|
||||
static std::string IKey(const std::string& user_key,
|
||||
uint64_t seq,
|
||||
ValueType vt) {
|
||||
std::string encoded;
|
||||
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
|
||||
@ -28,7 +28,9 @@ static std::string ShortSuccessor(const std::string& s) {
|
||||
return result;
|
||||
}
|
||||
|
||||
static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
|
||||
static void TestKey(const std::string& key,
|
||||
uint64_t seq,
|
||||
ValueType vt) {
|
||||
std::string encoded = IKey(key, seq, vt);
|
||||
|
||||
Slice in(encoded);
|
||||
@ -42,20 +44,16 @@ static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
|
||||
ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded));
|
||||
}
|
||||
|
||||
class FormatTest { };
|
||||
|
||||
TEST(FormatTest, InternalKey_EncodeDecode) {
|
||||
const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
|
||||
const uint64_t seq[] = {1,
|
||||
2,
|
||||
3,
|
||||
(1ull << 8) - 1,
|
||||
1ull << 8,
|
||||
(1ull << 8) + 1,
|
||||
(1ull << 16) - 1,
|
||||
1ull << 16,
|
||||
(1ull << 16) + 1,
|
||||
(1ull << 32) - 1,
|
||||
1ull << 32,
|
||||
(1ull << 32) + 1};
|
||||
const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" };
|
||||
const uint64_t seq[] = {
|
||||
1, 2, 3,
|
||||
(1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1,
|
||||
(1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1,
|
||||
(1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1
|
||||
};
|
||||
for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
|
||||
for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
|
||||
TestKey(keys[k], seq[s], kTypeValue);
|
||||
@ -64,44 +62,40 @@ TEST(FormatTest, InternalKey_EncodeDecode) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(FormatTest, InternalKey_DecodeFromEmpty) {
|
||||
InternalKey internal_key;
|
||||
|
||||
ASSERT_TRUE(!internal_key.DecodeFrom(""));
|
||||
}
|
||||
|
||||
TEST(FormatTest, InternalKeyShortSeparator) {
|
||||
// When user keys are same
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
|
||||
ASSERT_EQ(
|
||||
IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
|
||||
ASSERT_EQ(
|
||||
IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
|
||||
ASSERT_EQ(
|
||||
IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 99, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 101, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 100, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foo", 100, kTypeDeletion)));
|
||||
|
||||
// When user keys are misordered
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("bar", 99, kTypeValue)));
|
||||
|
||||
// When user keys are different, but correctly ordered
|
||||
ASSERT_EQ(
|
||||
IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
|
||||
ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("hello", 200, kTypeValue)));
|
||||
|
||||
// When start user key is prefix of limit user key
|
||||
ASSERT_EQ(
|
||||
IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foo", 100, kTypeValue),
|
||||
Shorten(IKey("foo", 100, kTypeValue),
|
||||
IKey("foobar", 200, kTypeValue)));
|
||||
|
||||
// When limit user key is prefix of start user key
|
||||
ASSERT_EQ(
|
||||
IKey("foobar", 100, kTypeValue),
|
||||
Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
|
||||
ASSERT_EQ(IKey("foobar", 100, kTypeValue),
|
||||
Shorten(IKey("foobar", 100, kTypeValue),
|
||||
IKey("foo", 200, kTypeValue)));
|
||||
}
|
||||
|
||||
TEST(FormatTest, InternalKeyShortestSuccessor) {
|
||||
@ -111,18 +105,8 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
|
||||
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
|
||||
}
|
||||
|
||||
TEST(FormatTest, ParsedInternalKeyDebugString) {
|
||||
ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue);
|
||||
|
||||
ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString());
|
||||
}
|
||||
|
||||
TEST(FormatTest, InternalKeyDebugString) {
|
||||
InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue);
|
||||
ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString());
|
||||
|
||||
InternalKey invalid_key;
|
||||
ASSERT_EQ("(bad)", invalid_key.DebugString());
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -4,7 +4,7 @@
|
||||
|
||||
#include "leveldb/dumpfile.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
@ -38,7 +38,8 @@ bool GuessType(const std::string& fname, FileType* type) {
|
||||
// Notified when log reader encounters corruption.
|
||||
class CorruptionReporter : public log::Reader::Reporter {
|
||||
public:
|
||||
void Corruption(size_t bytes, const Status& status) override {
|
||||
WritableFile* dst_;
|
||||
virtual void Corruption(size_t bytes, const Status& status) {
|
||||
std::string r = "corruption: ";
|
||||
AppendNumberTo(&r, bytes);
|
||||
r += " bytes; ";
|
||||
@ -46,8 +47,6 @@ class CorruptionReporter : public log::Reader::Reporter {
|
||||
r.push_back('\n');
|
||||
dst_->Append(r);
|
||||
}
|
||||
|
||||
WritableFile* dst_;
|
||||
};
|
||||
|
||||
// Print contents of a log file. (*func)() is called on every record.
|
||||
@ -74,7 +73,8 @@ Status PrintLogContents(Env* env, const std::string& fname,
|
||||
// Called on every item found in a WriteBatch.
|
||||
class WriteBatchItemPrinter : public WriteBatch::Handler {
|
||||
public:
|
||||
void Put(const Slice& key, const Slice& value) override {
|
||||
WritableFile* dst_;
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
std::string r = " put '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "' '";
|
||||
@ -82,16 +82,15 @@ class WriteBatchItemPrinter : public WriteBatch::Handler {
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
void Delete(const Slice& key) override {
|
||||
virtual void Delete(const Slice& key) {
|
||||
std::string r = " del '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
|
||||
WritableFile* dst_;
|
||||
};
|
||||
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kLogFile.
|
||||
static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
@ -217,12 +216,9 @@ Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return Status::InvalidArgument(fname + ": unknown file type");
|
||||
}
|
||||
switch (ftype) {
|
||||
case kLogFile:
|
||||
return DumpLog(env, fname, dst);
|
||||
case kDescriptorFile:
|
||||
return DumpDescriptor(env, fname, dst);
|
||||
case kTableFile:
|
||||
return DumpTable(env, fname, dst);
|
||||
case kLogFile: return DumpLog(env, fname, dst);
|
||||
case kDescriptorFile: return DumpDescriptor(env, fname, dst);
|
||||
case kTableFile: return DumpTable(env, fname, dst);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
@ -9,13 +9,12 @@
|
||||
#include <map>
|
||||
#include <set>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_format.h"
|
||||
#include "db/version_set.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
@ -23,6 +22,7 @@
|
||||
#include "port/thread_annotations.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -56,7 +56,8 @@ Status Truncate(const std::string& filename, uint64_t length) {
|
||||
|
||||
SequentialFile* orig_file;
|
||||
Status s = env->NewSequentialFile(filename, &orig_file);
|
||||
if (!s.ok()) return s;
|
||||
if (!s.ok())
|
||||
return s;
|
||||
|
||||
char* scratch = new char[length];
|
||||
leveldb::Slice result;
|
||||
@ -72,7 +73,7 @@ Status Truncate(const std::string& filename, uint64_t length) {
|
||||
if (s.ok()) {
|
||||
s = env->RenameFile(tmp_name, filename);
|
||||
} else {
|
||||
env->RemoveFile(tmp_name);
|
||||
env->DeleteFile(tmp_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -92,7 +93,7 @@ struct FileState {
|
||||
: filename_(filename),
|
||||
pos_(-1),
|
||||
pos_at_last_sync_(-1),
|
||||
pos_at_last_flush_(-1) {}
|
||||
pos_at_last_flush_(-1) { }
|
||||
|
||||
FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
|
||||
|
||||
@ -107,13 +108,14 @@ struct FileState {
|
||||
// is written to or sync'ed.
|
||||
class TestWritableFile : public WritableFile {
|
||||
public:
|
||||
TestWritableFile(const FileState& state, WritableFile* f,
|
||||
TestWritableFile(const FileState& state,
|
||||
WritableFile* f,
|
||||
FaultInjectionTestEnv* env);
|
||||
~TestWritableFile() override;
|
||||
Status Append(const Slice& data) override;
|
||||
Status Close() override;
|
||||
Status Flush() override;
|
||||
Status Sync() override;
|
||||
virtual ~TestWritableFile();
|
||||
virtual Status Append(const Slice& data);
|
||||
virtual Status Close();
|
||||
virtual Status Flush();
|
||||
virtual Status Sync();
|
||||
|
||||
private:
|
||||
FileState state_;
|
||||
@ -128,17 +130,17 @@ class FaultInjectionTestEnv : public EnvWrapper {
|
||||
public:
|
||||
FaultInjectionTestEnv()
|
||||
: EnvWrapper(Env::Default()), filesystem_active_(true) {}
|
||||
~FaultInjectionTestEnv() override = default;
|
||||
Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result) override;
|
||||
Status NewAppendableFile(const std::string& fname,
|
||||
WritableFile** result) override;
|
||||
Status RemoveFile(const std::string& f) override;
|
||||
Status RenameFile(const std::string& s, const std::string& t) override;
|
||||
virtual ~FaultInjectionTestEnv() { }
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result);
|
||||
virtual Status NewAppendableFile(const std::string& fname,
|
||||
WritableFile** result);
|
||||
virtual Status DeleteFile(const std::string& f);
|
||||
virtual Status RenameFile(const std::string& s, const std::string& t);
|
||||
|
||||
void WritableFileClosed(const FileState& state);
|
||||
Status DropUnsyncedFileData();
|
||||
Status RemoveFilesCreatedAfterLastDirSync();
|
||||
Status DeleteFilesCreatedAfterLastDirSync();
|
||||
void DirWasSynced();
|
||||
bool IsFileCreatedSinceLastDirSync(const std::string& filename);
|
||||
void ResetState();
|
||||
@ -163,9 +165,13 @@ class FaultInjectionTestEnv : public EnvWrapper {
|
||||
bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes
|
||||
};
|
||||
|
||||
TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f,
|
||||
TestWritableFile::TestWritableFile(const FileState& state,
|
||||
WritableFile* f,
|
||||
FaultInjectionTestEnv* env)
|
||||
: state_(state), target_(f), writable_file_opened_(true), env_(env) {
|
||||
: state_(state),
|
||||
target_(f),
|
||||
writable_file_opened_(true),
|
||||
env_(env) {
|
||||
assert(f != nullptr);
|
||||
}
|
||||
|
||||
@ -268,11 +274,10 @@ Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
|
||||
Status FaultInjectionTestEnv::DropUnsyncedFileData() {
|
||||
Status s;
|
||||
MutexLock l(&mutex_);
|
||||
for (const auto& kvp : db_file_state_) {
|
||||
if (!s.ok()) {
|
||||
break;
|
||||
}
|
||||
const FileState& state = kvp.second;
|
||||
for (std::map<std::string, FileState>::const_iterator it =
|
||||
db_file_state_.begin();
|
||||
s.ok() && it != db_file_state_.end(); ++it) {
|
||||
const FileState& state = it->second;
|
||||
if (!state.IsFullySynced()) {
|
||||
s = state.DropUnsyncedData();
|
||||
}
|
||||
@ -298,9 +303,9 @@ void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
|
||||
new_files_since_last_dir_sync_.erase(f);
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::RemoveFile(const std::string& f) {
|
||||
Status s = EnvWrapper::RemoveFile(f);
|
||||
EXPECT_LEVELDB_OK(s);
|
||||
Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
|
||||
Status s = EnvWrapper::DeleteFile(f);
|
||||
ASSERT_OK(s);
|
||||
if (s.ok()) {
|
||||
UntrackFile(f);
|
||||
}
|
||||
@ -335,20 +340,18 @@ void FaultInjectionTestEnv::ResetState() {
|
||||
SetFilesystemActive(true);
|
||||
}
|
||||
|
||||
Status FaultInjectionTestEnv::RemoveFilesCreatedAfterLastDirSync() {
|
||||
// Because RemoveFile access this container make a copy to avoid deadlock
|
||||
Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
|
||||
// Because DeleteFile access this container make a copy to avoid deadlock
|
||||
mutex_.Lock();
|
||||
std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
|
||||
new_files_since_last_dir_sync_.end());
|
||||
mutex_.Unlock();
|
||||
Status status;
|
||||
for (const auto& new_file : new_files) {
|
||||
Status remove_status = RemoveFile(new_file);
|
||||
if (!remove_status.ok() && status.ok()) {
|
||||
status = std::move(remove_status);
|
||||
}
|
||||
Status s;
|
||||
std::set<std::string>::const_iterator it;
|
||||
for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) {
|
||||
s = DeleteFile(*it);
|
||||
}
|
||||
return status;
|
||||
return s;
|
||||
}
|
||||
|
||||
void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
|
||||
@ -361,7 +364,7 @@ Status FileState::DropUnsyncedData() const {
|
||||
return Truncate(filename_, sync_pos);
|
||||
}
|
||||
|
||||
class FaultInjectionTest : public testing::Test {
|
||||
class FaultInjectionTest {
|
||||
public:
|
||||
enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR };
|
||||
enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES };
|
||||
@ -376,7 +379,7 @@ class FaultInjectionTest : public testing::Test {
|
||||
: env_(new FaultInjectionTestEnv),
|
||||
tiny_cache_(NewLRUCache(100)),
|
||||
db_(nullptr) {
|
||||
dbname_ = testing::TempDir() + "fault_test";
|
||||
dbname_ = test::TmpDir() + "/fault_test";
|
||||
DestroyDB(dbname_, Options()); // Destroy any db from earlier run
|
||||
options_.reuse_logs = true;
|
||||
options_.env = env_;
|
||||
@ -392,7 +395,9 @@ class FaultInjectionTest : public testing::Test {
|
||||
delete env_;
|
||||
}
|
||||
|
||||
void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; }
|
||||
void ReuseLogs(bool reuse) {
|
||||
options_.reuse_logs = reuse;
|
||||
}
|
||||
|
||||
void Build(int start_idx, int num_vals) {
|
||||
std::string key_space, value_space;
|
||||
@ -402,7 +407,7 @@ class FaultInjectionTest : public testing::Test {
|
||||
batch.Clear();
|
||||
batch.Put(key, Value(i, &value_space));
|
||||
WriteOptions options;
|
||||
ASSERT_LEVELDB_OK(db_->Write(options, &batch));
|
||||
ASSERT_OK(db_->Write(options, &batch));
|
||||
}
|
||||
}
|
||||
|
||||
@ -424,10 +429,10 @@ class FaultInjectionTest : public testing::Test {
|
||||
s = ReadValue(i, &val);
|
||||
if (expected == VAL_EXPECT_NO_ERROR) {
|
||||
if (s.ok()) {
|
||||
EXPECT_EQ(value_space, val);
|
||||
ASSERT_EQ(value_space, val);
|
||||
}
|
||||
} else if (s.ok()) {
|
||||
std::fprintf(stderr, "Expected an error at %d, but was OK\n", i);
|
||||
fprintf(stderr, "Expected an error at %d, but was OK\n", i);
|
||||
s = Status::IOError(dbname_, "Expected value error:");
|
||||
} else {
|
||||
s = Status::OK(); // An expected error
|
||||
@ -439,7 +444,7 @@ class FaultInjectionTest : public testing::Test {
|
||||
// Return the ith key
|
||||
Slice Key(int i, std::string* storage) const {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "%016d", i);
|
||||
snprintf(buf, sizeof(buf), "%016d", i);
|
||||
storage->assign(buf, strlen(buf));
|
||||
return Slice(*storage);
|
||||
}
|
||||
@ -465,7 +470,7 @@ class FaultInjectionTest : public testing::Test {
|
||||
void DeleteAllData() {
|
||||
Iterator* iter = db_->NewIterator(ReadOptions());
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), iter->key()));
|
||||
ASSERT_OK(db_->Delete(WriteOptions(), iter->key()));
|
||||
}
|
||||
|
||||
delete iter;
|
||||
@ -474,10 +479,10 @@ class FaultInjectionTest : public testing::Test {
|
||||
void ResetDBState(ResetMethod reset_method) {
|
||||
switch (reset_method) {
|
||||
case RESET_DROP_UNSYNCED_DATA:
|
||||
ASSERT_LEVELDB_OK(env_->DropUnsyncedFileData());
|
||||
ASSERT_OK(env_->DropUnsyncedFileData());
|
||||
break;
|
||||
case RESET_DELETE_UNSYNCED_FILES:
|
||||
ASSERT_LEVELDB_OK(env_->RemoveFilesCreatedAfterLastDirSync());
|
||||
ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync());
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
@ -492,34 +497,35 @@ class FaultInjectionTest : public testing::Test {
|
||||
}
|
||||
|
||||
void PartialCompactTestReopenWithFault(ResetMethod reset_method,
|
||||
int num_pre_sync, int num_post_sync) {
|
||||
int num_pre_sync,
|
||||
int num_post_sync) {
|
||||
env_->SetFilesystemActive(false);
|
||||
CloseDB();
|
||||
ResetDBState(reset_method);
|
||||
ASSERT_LEVELDB_OK(OpenDB());
|
||||
ASSERT_LEVELDB_OK(
|
||||
Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
|
||||
ASSERT_LEVELDB_OK(Verify(num_pre_sync, num_post_sync,
|
||||
FaultInjectionTest::VAL_EXPECT_ERROR));
|
||||
ASSERT_OK(OpenDB());
|
||||
ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
|
||||
ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR));
|
||||
}
|
||||
|
||||
void NoWriteTestPreFault() {}
|
||||
void NoWriteTestPreFault() {
|
||||
}
|
||||
|
||||
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
|
||||
CloseDB();
|
||||
ResetDBState(reset_method);
|
||||
ASSERT_LEVELDB_OK(OpenDB());
|
||||
ASSERT_OK(OpenDB());
|
||||
}
|
||||
|
||||
void DoTest() {
|
||||
Random rnd(0);
|
||||
ASSERT_LEVELDB_OK(OpenDB());
|
||||
ASSERT_OK(OpenDB());
|
||||
for (size_t idx = 0; idx < kNumIterations; idx++) {
|
||||
int num_pre_sync = rnd.Uniform(kMaxNumValues);
|
||||
int num_post_sync = rnd.Uniform(kMaxNumValues);
|
||||
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync);
|
||||
PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync,
|
||||
PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA,
|
||||
num_pre_sync,
|
||||
num_post_sync);
|
||||
|
||||
NoWriteTestPreFault();
|
||||
@ -529,7 +535,8 @@ class FaultInjectionTest : public testing::Test {
|
||||
// No new files created so we expect all values since no files will be
|
||||
// dropped.
|
||||
PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
|
||||
num_pre_sync + num_post_sync, 0);
|
||||
num_pre_sync + num_post_sync,
|
||||
0);
|
||||
|
||||
NoWriteTestPreFault();
|
||||
NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
|
||||
@ -537,14 +544,18 @@ class FaultInjectionTest : public testing::Test {
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(FaultInjectionTest, FaultTestNoLogReuse) {
|
||||
TEST(FaultInjectionTest, FaultTestNoLogReuse) {
|
||||
ReuseLogs(false);
|
||||
DoTest();
|
||||
}
|
||||
|
||||
TEST_F(FaultInjectionTest, FaultTestWithLogReuse) {
|
||||
TEST(FaultInjectionTest, FaultTestWithLogReuse) {
|
||||
ReuseLogs(true);
|
||||
DoTest();
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -2,11 +2,9 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
#include "db/filename.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/logging.h"
|
||||
@ -20,8 +18,9 @@ Status WriteStringToFileSync(Env* env, const Slice& data,
|
||||
static std::string MakeFileName(const std::string& dbname, uint64_t number,
|
||||
const char* suffix) {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "/%06llu.%s",
|
||||
static_cast<unsigned long long>(number), suffix);
|
||||
snprintf(buf, sizeof(buf), "/%06llu.%s",
|
||||
static_cast<unsigned long long>(number),
|
||||
suffix);
|
||||
return dbname + buf;
|
||||
}
|
||||
|
||||
@ -43,8 +42,8 @@ std::string SSTTableFileName(const std::string& dbname, uint64_t number) {
|
||||
std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "/MANIFEST-%06llu",
|
||||
static_cast<unsigned long long>(number));
|
||||
snprintf(buf, sizeof(buf), "/MANIFEST-%06llu",
|
||||
static_cast<unsigned long long>(number));
|
||||
return dbname + buf;
|
||||
}
|
||||
|
||||
@ -52,7 +51,9 @@ std::string CurrentFileName(const std::string& dbname) {
|
||||
return dbname + "/CURRENT";
|
||||
}
|
||||
|
||||
std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; }
|
||||
std::string LockFileName(const std::string& dbname) {
|
||||
return dbname + "/LOCK";
|
||||
}
|
||||
|
||||
std::string TempFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number > 0);
|
||||
@ -68,6 +69,7 @@ std::string OldInfoLogFileName(const std::string& dbname) {
|
||||
return dbname + "/LOG.old";
|
||||
}
|
||||
|
||||
|
||||
// Owned filenames have the form:
|
||||
// dbname/CURRENT
|
||||
// dbname/LOCK
|
||||
@ -75,7 +77,8 @@ std::string OldInfoLogFileName(const std::string& dbname) {
|
||||
// dbname/LOG.old
|
||||
// dbname/MANIFEST-[0-9]+
|
||||
// dbname/[0-9]+.(log|sst|ldb)
|
||||
bool ParseFileName(const std::string& filename, uint64_t* number,
|
||||
bool ParseFileName(const std::string& filename,
|
||||
uint64_t* number,
|
||||
FileType* type) {
|
||||
Slice rest(filename);
|
||||
if (rest == "CURRENT") {
|
||||
@ -133,7 +136,7 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
s = env->RenameFile(tmp, CurrentFileName(dbname));
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env->RemoveFile(tmp);
|
||||
env->DeleteFile(tmp);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
@ -7,9 +7,8 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
#define STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "port/port.h"
|
||||
@ -70,7 +69,8 @@ std::string OldInfoLogFileName(const std::string& dbname);
|
||||
// If filename is a leveldb file, store the type of the file in *type.
|
||||
// The number encoded in the filename is stored in *number. If the
|
||||
// filename was successfully parsed, returns true. Else return false.
|
||||
bool ParseFileName(const std::string& filename, uint64_t* number,
|
||||
bool ParseFileName(const std::string& filename,
|
||||
uint64_t* number,
|
||||
FileType* type);
|
||||
|
||||
// Make the CURRENT file point to the descriptor file with the
|
||||
|
@ -4,13 +4,15 @@
|
||||
|
||||
#include "db/filename.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "port/port.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class FileNameTest { };
|
||||
|
||||
TEST(FileNameTest, Parse) {
|
||||
Slice db;
|
||||
FileType type;
|
||||
@ -22,17 +24,17 @@ TEST(FileNameTest, Parse) {
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
} cases[] = {
|
||||
{"100.log", 100, kLogFile},
|
||||
{"0.log", 0, kLogFile},
|
||||
{"0.sst", 0, kTableFile},
|
||||
{"0.ldb", 0, kTableFile},
|
||||
{"CURRENT", 0, kCurrentFile},
|
||||
{"LOCK", 0, kDBLockFile},
|
||||
{"MANIFEST-2", 2, kDescriptorFile},
|
||||
{"MANIFEST-7", 7, kDescriptorFile},
|
||||
{"LOG", 0, kInfoLogFile},
|
||||
{"LOG.old", 0, kInfoLogFile},
|
||||
{"18446744073709551615.log", 18446744073709551615ull, kLogFile},
|
||||
{ "100.log", 100, kLogFile },
|
||||
{ "0.log", 0, kLogFile },
|
||||
{ "0.sst", 0, kTableFile },
|
||||
{ "0.ldb", 0, kTableFile },
|
||||
{ "CURRENT", 0, kCurrentFile },
|
||||
{ "LOCK", 0, kDBLockFile },
|
||||
{ "MANIFEST-2", 2, kDescriptorFile },
|
||||
{ "MANIFEST-7", 7, kDescriptorFile },
|
||||
{ "LOG", 0, kInfoLogFile },
|
||||
{ "LOG.old", 0, kInfoLogFile },
|
||||
{ "18446744073709551615.log", 18446744073709551615ull, kLogFile },
|
||||
};
|
||||
for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
std::string f = cases[i].fname;
|
||||
@ -42,28 +44,30 @@ TEST(FileNameTest, Parse) {
|
||||
}
|
||||
|
||||
// Errors
|
||||
static const char* errors[] = {"",
|
||||
"foo",
|
||||
"foo-dx-100.log",
|
||||
".log",
|
||||
"",
|
||||
"manifest",
|
||||
"CURREN",
|
||||
"CURRENTX",
|
||||
"MANIFES",
|
||||
"MANIFEST",
|
||||
"MANIFEST-",
|
||||
"XMANIFEST-3",
|
||||
"MANIFEST-3x",
|
||||
"LOC",
|
||||
"LOCKx",
|
||||
"LO",
|
||||
"LOGx",
|
||||
"18446744073709551616.log",
|
||||
"184467440737095516150.log",
|
||||
"100",
|
||||
"100.",
|
||||
"100.lop"};
|
||||
static const char* errors[] = {
|
||||
"",
|
||||
"foo",
|
||||
"foo-dx-100.log",
|
||||
".log",
|
||||
"",
|
||||
"manifest",
|
||||
"CURREN",
|
||||
"CURRENTX",
|
||||
"MANIFES",
|
||||
"MANIFEST",
|
||||
"MANIFEST-",
|
||||
"XMANIFEST-3",
|
||||
"MANIFEST-3x",
|
||||
"LOC",
|
||||
"LOCKx",
|
||||
"LO",
|
||||
"LOGx",
|
||||
"18446744073709551616.log",
|
||||
"184467440737095516150.log",
|
||||
"100",
|
||||
"100.",
|
||||
"100.lop"
|
||||
};
|
||||
for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
|
||||
std::string f = errors[i];
|
||||
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
|
||||
@ -125,3 +129,7 @@ TEST(FileNameTest, Construction) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -2,8 +2,7 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include <stdio.h>
|
||||
#include "leveldb/dumpfile.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
@ -13,13 +12,13 @@ namespace {
|
||||
|
||||
class StdoutPrinter : public WritableFile {
|
||||
public:
|
||||
Status Append(const Slice& data) override {
|
||||
virtual Status Append(const Slice& data) {
|
||||
fwrite(data.data(), 1, data.size(), stdout);
|
||||
return Status::OK();
|
||||
}
|
||||
Status Close() override { return Status::OK(); }
|
||||
Status Flush() override { return Status::OK(); }
|
||||
Status Sync() override { return Status::OK(); }
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
};
|
||||
|
||||
bool HandleDumpCommand(Env* env, char** files, int num) {
|
||||
@ -28,7 +27,7 @@ bool HandleDumpCommand(Env* env, char** files, int num) {
|
||||
for (int i = 0; i < num; i++) {
|
||||
Status s = DumpFile(env, files[i], &printer);
|
||||
if (!s.ok()) {
|
||||
std::fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
@ -39,10 +38,11 @@ bool HandleDumpCommand(Env* env, char** files, int num) {
|
||||
} // namespace leveldb
|
||||
|
||||
static void Usage() {
|
||||
std::fprintf(
|
||||
fprintf(
|
||||
stderr,
|
||||
"Usage: leveldbutil command...\n"
|
||||
" dump files... -- dump contents of specified files\n");
|
||||
" dump files... -- dump contents of specified files\n"
|
||||
);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
@ -54,7 +54,7 @@ int main(int argc, char** argv) {
|
||||
} else {
|
||||
std::string command = argv[1];
|
||||
if (command == "dump") {
|
||||
ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2);
|
||||
ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
|
||||
} else {
|
||||
Usage();
|
||||
ok = false;
|
||||
|
@ -4,8 +4,7 @@
|
||||
|
||||
#include "db/log_reader.h"
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include <stdio.h>
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
@ -13,7 +12,8 @@
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
Reader::Reporter::~Reporter() = default;
|
||||
Reader::Reporter::~Reporter() {
|
||||
}
|
||||
|
||||
Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
|
||||
uint64_t initial_offset)
|
||||
@ -26,9 +26,12 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
|
||||
last_record_offset_(0),
|
||||
end_of_buffer_offset_(0),
|
||||
initial_offset_(initial_offset),
|
||||
resyncing_(initial_offset > 0) {}
|
||||
resyncing_(initial_offset > 0) {
|
||||
}
|
||||
|
||||
Reader::~Reader() { delete[] backing_store_; }
|
||||
Reader::~Reader() {
|
||||
delete[] backing_store_;
|
||||
}
|
||||
|
||||
bool Reader::SkipToInitialBlock() {
|
||||
const size_t offset_in_block = initial_offset_ % kBlockSize;
|
||||
@ -160,7 +163,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
|
||||
|
||||
default: {
|
||||
char buf[40];
|
||||
std::snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
|
||||
snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
|
||||
ReportCorruption(
|
||||
(fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
|
||||
buf);
|
||||
@ -173,7 +176,9 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
|
||||
uint64_t Reader::LastRecordOffset() {
|
||||
return last_record_offset_;
|
||||
}
|
||||
|
||||
void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
|
||||
ReportDrop(bytes, Status::Corruption(reason));
|
||||
|
@ -5,7 +5,7 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "db/log_format.h"
|
||||
#include "leveldb/slice.h"
|
||||
@ -24,7 +24,7 @@ class Reader {
|
||||
public:
|
||||
virtual ~Reporter();
|
||||
|
||||
// Some corruption was detected. "bytes" is the approximate number
|
||||
// Some corruption was detected. "size" is the approximate number
|
||||
// of bytes dropped due to the corruption.
|
||||
virtual void Corruption(size_t bytes, const Status& status) = 0;
|
||||
};
|
||||
@ -43,9 +43,6 @@ class Reader {
|
||||
Reader(SequentialFile* file, Reporter* reporter, bool checksum,
|
||||
uint64_t initial_offset);
|
||||
|
||||
Reader(const Reader&) = delete;
|
||||
Reader& operator=(const Reader&) = delete;
|
||||
|
||||
~Reader();
|
||||
|
||||
// Read the next record into *record. Returns true if read
|
||||
@ -61,6 +58,26 @@ class Reader {
|
||||
uint64_t LastRecordOffset();
|
||||
|
||||
private:
|
||||
SequentialFile* const file_;
|
||||
Reporter* const reporter_;
|
||||
bool const checksum_;
|
||||
char* const backing_store_;
|
||||
Slice buffer_;
|
||||
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
|
||||
|
||||
// Offset of the last record returned by ReadRecord.
|
||||
uint64_t last_record_offset_;
|
||||
// Offset of the first location past the end of buffer_.
|
||||
uint64_t end_of_buffer_offset_;
|
||||
|
||||
// Offset at which to start looking for the first record to return
|
||||
uint64_t const initial_offset_;
|
||||
|
||||
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
|
||||
// particular, a run of kMiddleType and kLastType records can be silently
|
||||
// skipped in this mode
|
||||
bool resyncing_;
|
||||
|
||||
// Extend record types with the following special values
|
||||
enum {
|
||||
kEof = kMaxRecordType + 1,
|
||||
@ -85,25 +102,9 @@ class Reader {
|
||||
void ReportCorruption(uint64_t bytes, const char* reason);
|
||||
void ReportDrop(uint64_t bytes, const Status& reason);
|
||||
|
||||
SequentialFile* const file_;
|
||||
Reporter* const reporter_;
|
||||
bool const checksum_;
|
||||
char* const backing_store_;
|
||||
Slice buffer_;
|
||||
bool eof_; // Last Read() indicated EOF by returning < kBlockSize
|
||||
|
||||
// Offset of the last record returned by ReadRecord.
|
||||
uint64_t last_record_offset_;
|
||||
// Offset of the first location past the end of buffer_.
|
||||
uint64_t end_of_buffer_offset_;
|
||||
|
||||
// Offset at which to start looking for the first record to return
|
||||
uint64_t const initial_offset_;
|
||||
|
||||
// True if we are resynchronizing after a seek (initial_offset_ > 0). In
|
||||
// particular, a run of kMiddleType and kLastType records can be silently
|
||||
// skipped in this mode
|
||||
bool resyncing_;
|
||||
// No copying allowed
|
||||
Reader(const Reader&);
|
||||
void operator=(const Reader&);
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
|
349
db/log_test.cc
349
db/log_test.cc
@ -2,13 +2,13 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
@ -27,7 +27,7 @@ static std::string BigString(const std::string& partial_string, size_t n) {
|
||||
// Construct a string from a number
|
||||
static std::string NumberString(int n) {
|
||||
char buf[50];
|
||||
std::snprintf(buf, sizeof(buf), "%d.", n);
|
||||
snprintf(buf, sizeof(buf), "%d.", n);
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
@ -36,13 +36,88 @@ static std::string RandomSkewedString(int i, Random* rnd) {
|
||||
return BigString(NumberString(i), rnd->Skewed(17));
|
||||
}
|
||||
|
||||
class LogTest : public testing::Test {
|
||||
class LogTest {
|
||||
private:
|
||||
class StringDest : public WritableFile {
|
||||
public:
|
||||
std::string contents_;
|
||||
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
virtual Status Append(const Slice& slice) {
|
||||
contents_.append(slice.data(), slice.size());
|
||||
return Status::OK();
|
||||
}
|
||||
};
|
||||
|
||||
class StringSource : public SequentialFile {
|
||||
public:
|
||||
Slice contents_;
|
||||
bool force_error_;
|
||||
bool returned_partial_;
|
||||
StringSource() : force_error_(false), returned_partial_(false) { }
|
||||
|
||||
virtual Status Read(size_t n, Slice* result, char* scratch) {
|
||||
ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
|
||||
|
||||
if (force_error_) {
|
||||
force_error_ = false;
|
||||
returned_partial_ = true;
|
||||
return Status::Corruption("read error");
|
||||
}
|
||||
|
||||
if (contents_.size() < n) {
|
||||
n = contents_.size();
|
||||
returned_partial_ = true;
|
||||
}
|
||||
*result = Slice(contents_.data(), n);
|
||||
contents_.remove_prefix(n);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
virtual Status Skip(uint64_t n) {
|
||||
if (n > contents_.size()) {
|
||||
contents_.clear();
|
||||
return Status::NotFound("in-memory file skipped past end");
|
||||
}
|
||||
|
||||
contents_.remove_prefix(n);
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
};
|
||||
|
||||
class ReportCollector : public Reader::Reporter {
|
||||
public:
|
||||
size_t dropped_bytes_;
|
||||
std::string message_;
|
||||
|
||||
ReportCollector() : dropped_bytes_(0) { }
|
||||
virtual void Corruption(size_t bytes, const Status& status) {
|
||||
dropped_bytes_ += bytes;
|
||||
message_.append(status.ToString());
|
||||
}
|
||||
};
|
||||
|
||||
StringDest dest_;
|
||||
StringSource source_;
|
||||
ReportCollector report_;
|
||||
bool reading_;
|
||||
Writer* writer_;
|
||||
Reader* reader_;
|
||||
|
||||
// Record metadata for testing initial offset functionality
|
||||
static size_t initial_offset_record_sizes_[];
|
||||
static uint64_t initial_offset_last_record_offsets_[];
|
||||
static int num_initial_offset_records_;
|
||||
|
||||
public:
|
||||
LogTest()
|
||||
: reading_(false),
|
||||
writer_(new Writer(&dest_)),
|
||||
reader_(new Reader(&source_, &report_, true /*checksum*/,
|
||||
0 /*initial_offset*/)) {}
|
||||
LogTest() : reading_(false),
|
||||
writer_(new Writer(&dest_)),
|
||||
reader_(new Reader(&source_, &report_, true/*checksum*/,
|
||||
0/*initial_offset*/)) {
|
||||
}
|
||||
|
||||
~LogTest() {
|
||||
delete writer_;
|
||||
@ -59,7 +134,9 @@ class LogTest : public testing::Test {
|
||||
writer_->AddRecord(Slice(msg));
|
||||
}
|
||||
|
||||
size_t WrittenBytes() const { return dest_.contents_.size(); }
|
||||
size_t WrittenBytes() const {
|
||||
return dest_.contents_.size();
|
||||
}
|
||||
|
||||
std::string Read() {
|
||||
if (!reading_) {
|
||||
@ -89,16 +166,22 @@ class LogTest : public testing::Test {
|
||||
|
||||
void FixChecksum(int header_offset, int len) {
|
||||
// Compute crc of type/len/data
|
||||
uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len);
|
||||
uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len);
|
||||
crc = crc32c::Mask(crc);
|
||||
EncodeFixed32(&dest_.contents_[header_offset], crc);
|
||||
}
|
||||
|
||||
void ForceError() { source_.force_error_ = true; }
|
||||
void ForceError() {
|
||||
source_.force_error_ = true;
|
||||
}
|
||||
|
||||
size_t DroppedBytes() const { return report_.dropped_bytes_; }
|
||||
size_t DroppedBytes() const {
|
||||
return report_.dropped_bytes_;
|
||||
}
|
||||
|
||||
std::string ReportMessage() const { return report_.message_; }
|
||||
std::string ReportMessage() const {
|
||||
return report_.message_;
|
||||
}
|
||||
|
||||
// Returns OK iff recorded error message contains "msg"
|
||||
std::string MatchError(const std::string& msg) const {
|
||||
@ -119,14 +202,14 @@ class LogTest : public testing::Test {
|
||||
|
||||
void StartReadingAt(uint64_t initial_offset) {
|
||||
delete reader_;
|
||||
reader_ = new Reader(&source_, &report_, true /*checksum*/, initial_offset);
|
||||
reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset);
|
||||
}
|
||||
|
||||
void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
|
||||
WriteInitialOffsetLog();
|
||||
reading_ = true;
|
||||
source_.contents_ = Slice(dest_.contents_);
|
||||
Reader* offset_reader = new Reader(&source_, &report_, true /*checksum*/,
|
||||
Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
|
||||
WrittenBytes() + offset_past_end);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
@ -139,8 +222,8 @@ class LogTest : public testing::Test {
|
||||
WriteInitialOffsetLog();
|
||||
reading_ = true;
|
||||
source_.contents_ = Slice(dest_.contents_);
|
||||
Reader* offset_reader =
|
||||
new Reader(&source_, &report_, true /*checksum*/, initial_offset);
|
||||
Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
|
||||
initial_offset);
|
||||
|
||||
// Read all records from expected_record_offset through the last one.
|
||||
ASSERT_LT(expected_record_offset, num_initial_offset_records_);
|
||||
@ -157,110 +240,38 @@ class LogTest : public testing::Test {
|
||||
}
|
||||
delete offset_reader;
|
||||
}
|
||||
|
||||
private:
|
||||
class StringDest : public WritableFile {
|
||||
public:
|
||||
Status Close() override { return Status::OK(); }
|
||||
Status Flush() override { return Status::OK(); }
|
||||
Status Sync() override { return Status::OK(); }
|
||||
Status Append(const Slice& slice) override {
|
||||
contents_.append(slice.data(), slice.size());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::string contents_;
|
||||
};
|
||||
|
||||
class StringSource : public SequentialFile {
|
||||
public:
|
||||
StringSource() : force_error_(false), returned_partial_(false) {}
|
||||
|
||||
Status Read(size_t n, Slice* result, char* scratch) override {
|
||||
EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error";
|
||||
|
||||
if (force_error_) {
|
||||
force_error_ = false;
|
||||
returned_partial_ = true;
|
||||
return Status::Corruption("read error");
|
||||
}
|
||||
|
||||
if (contents_.size() < n) {
|
||||
n = contents_.size();
|
||||
returned_partial_ = true;
|
||||
}
|
||||
*result = Slice(contents_.data(), n);
|
||||
contents_.remove_prefix(n);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Skip(uint64_t n) override {
|
||||
if (n > contents_.size()) {
|
||||
contents_.clear();
|
||||
return Status::NotFound("in-memory file skipped past end");
|
||||
}
|
||||
|
||||
contents_.remove_prefix(n);
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Slice contents_;
|
||||
bool force_error_;
|
||||
bool returned_partial_;
|
||||
};
|
||||
|
||||
class ReportCollector : public Reader::Reporter {
|
||||
public:
|
||||
ReportCollector() : dropped_bytes_(0) {}
|
||||
void Corruption(size_t bytes, const Status& status) override {
|
||||
dropped_bytes_ += bytes;
|
||||
message_.append(status.ToString());
|
||||
}
|
||||
|
||||
size_t dropped_bytes_;
|
||||
std::string message_;
|
||||
};
|
||||
|
||||
// Record metadata for testing initial offset functionality
|
||||
static size_t initial_offset_record_sizes_[];
|
||||
static uint64_t initial_offset_last_record_offsets_[];
|
||||
static int num_initial_offset_records_;
|
||||
|
||||
StringDest dest_;
|
||||
StringSource source_;
|
||||
ReportCollector report_;
|
||||
bool reading_;
|
||||
Writer* writer_;
|
||||
Reader* reader_;
|
||||
};
|
||||
|
||||
size_t LogTest::initial_offset_record_sizes_[] = {
|
||||
10000, // Two sizable records in first block
|
||||
10000,
|
||||
2 * log::kBlockSize - 1000, // Span three blocks
|
||||
1,
|
||||
13716, // Consume all but two bytes of block 3.
|
||||
log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
|
||||
};
|
||||
size_t LogTest::initial_offset_record_sizes_[] =
|
||||
{10000, // Two sizable records in first block
|
||||
10000,
|
||||
2 * log::kBlockSize - 1000, // Span three blocks
|
||||
1,
|
||||
13716, // Consume all but two bytes of block 3.
|
||||
log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
|
||||
};
|
||||
|
||||
uint64_t LogTest::initial_offset_last_record_offsets_[] = {
|
||||
0,
|
||||
kHeaderSize + 10000,
|
||||
2 * (kHeaderSize + 10000),
|
||||
2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
|
||||
2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize +
|
||||
kHeaderSize + 1,
|
||||
3 * log::kBlockSize,
|
||||
};
|
||||
uint64_t LogTest::initial_offset_last_record_offsets_[] =
|
||||
{0,
|
||||
kHeaderSize + 10000,
|
||||
2 * (kHeaderSize + 10000),
|
||||
2 * (kHeaderSize + 10000) +
|
||||
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
|
||||
2 * (kHeaderSize + 10000) +
|
||||
(2 * log::kBlockSize - 1000) + 3 * kHeaderSize
|
||||
+ kHeaderSize + 1,
|
||||
3 * log::kBlockSize,
|
||||
};
|
||||
|
||||
// LogTest::initial_offset_last_record_offsets_ must be defined before this.
|
||||
int LogTest::num_initial_offset_records_ =
|
||||
sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t);
|
||||
sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t);
|
||||
|
||||
TEST_F(LogTest, Empty) { ASSERT_EQ("EOF", Read()); }
|
||||
TEST(LogTest, Empty) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadWrite) {
|
||||
TEST(LogTest, ReadWrite) {
|
||||
Write("foo");
|
||||
Write("bar");
|
||||
Write("");
|
||||
@ -273,7 +284,7 @@ TEST_F(LogTest, ReadWrite) {
|
||||
ASSERT_EQ("EOF", Read()); // Make sure reads at eof work
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ManyBlocks) {
|
||||
TEST(LogTest, ManyBlocks) {
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
Write(NumberString(i));
|
||||
}
|
||||
@ -283,7 +294,7 @@ TEST_F(LogTest, ManyBlocks) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, Fragmentation) {
|
||||
TEST(LogTest, Fragmentation) {
|
||||
Write("small");
|
||||
Write(BigString("medium", 50000));
|
||||
Write(BigString("large", 100000));
|
||||
@ -293,9 +304,9 @@ TEST_F(LogTest, Fragmentation) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, MarginalTrailer) {
|
||||
TEST(LogTest, MarginalTrailer) {
|
||||
// Make a trailer that is exactly the same length as an empty record.
|
||||
const int n = kBlockSize - 2 * kHeaderSize;
|
||||
const int n = kBlockSize - 2*kHeaderSize;
|
||||
Write(BigString("foo", n));
|
||||
ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
|
||||
Write("");
|
||||
@ -306,9 +317,9 @@ TEST_F(LogTest, MarginalTrailer) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, MarginalTrailer2) {
|
||||
TEST(LogTest, MarginalTrailer2) {
|
||||
// Make a trailer that is exactly the same length as an empty record.
|
||||
const int n = kBlockSize - 2 * kHeaderSize;
|
||||
const int n = kBlockSize - 2*kHeaderSize;
|
||||
Write(BigString("foo", n));
|
||||
ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
|
||||
Write("bar");
|
||||
@ -319,8 +330,8 @@ TEST_F(LogTest, MarginalTrailer2) {
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ShortTrailer) {
|
||||
const int n = kBlockSize - 2 * kHeaderSize + 4;
|
||||
TEST(LogTest, ShortTrailer) {
|
||||
const int n = kBlockSize - 2*kHeaderSize + 4;
|
||||
Write(BigString("foo", n));
|
||||
ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
|
||||
Write("");
|
||||
@ -331,15 +342,15 @@ TEST_F(LogTest, ShortTrailer) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, AlignedEof) {
|
||||
const int n = kBlockSize - 2 * kHeaderSize + 4;
|
||||
TEST(LogTest, AlignedEof) {
|
||||
const int n = kBlockSize - 2*kHeaderSize + 4;
|
||||
Write(BigString("foo", n));
|
||||
ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
|
||||
ASSERT_EQ(BigString("foo", n), Read());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, OpenForAppend) {
|
||||
TEST(LogTest, OpenForAppend) {
|
||||
Write("hello");
|
||||
ReopenForAppend();
|
||||
Write("world");
|
||||
@ -348,7 +359,7 @@ TEST_F(LogTest, OpenForAppend) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, RandomRead) {
|
||||
TEST(LogTest, RandomRead) {
|
||||
const int N = 500;
|
||||
Random write_rnd(301);
|
||||
for (int i = 0; i < N; i++) {
|
||||
@ -363,7 +374,7 @@ TEST_F(LogTest, RandomRead) {
|
||||
|
||||
// Tests of all the error paths in log_reader.cc follow:
|
||||
|
||||
TEST_F(LogTest, ReadError) {
|
||||
TEST(LogTest, ReadError) {
|
||||
Write("foo");
|
||||
ForceError();
|
||||
ASSERT_EQ("EOF", Read());
|
||||
@ -371,7 +382,7 @@ TEST_F(LogTest, ReadError) {
|
||||
ASSERT_EQ("OK", MatchError("read error"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, BadRecordType) {
|
||||
TEST(LogTest, BadRecordType) {
|
||||
Write("foo");
|
||||
// Type is stored in header[6]
|
||||
IncrementByte(6, 100);
|
||||
@ -381,16 +392,16 @@ TEST_F(LogTest, BadRecordType) {
|
||||
ASSERT_EQ("OK", MatchError("unknown record type"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, TruncatedTrailingRecordIsIgnored) {
|
||||
TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
|
||||
Write("foo");
|
||||
ShrinkSize(4); // Drop all payload as well as a header byte
|
||||
ShrinkSize(4); // Drop all payload as well as a header byte
|
||||
ASSERT_EQ("EOF", Read());
|
||||
// Truncated last record is ignored, not treated as an error.
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, BadLength) {
|
||||
TEST(LogTest, BadLength) {
|
||||
const int kPayloadSize = kBlockSize - kHeaderSize;
|
||||
Write(BigString("bar", kPayloadSize));
|
||||
Write("foo");
|
||||
@ -401,7 +412,7 @@ TEST_F(LogTest, BadLength) {
|
||||
ASSERT_EQ("OK", MatchError("bad record length"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, BadLengthAtEndIsIgnored) {
|
||||
TEST(LogTest, BadLengthAtEndIsIgnored) {
|
||||
Write("foo");
|
||||
ShrinkSize(1);
|
||||
ASSERT_EQ("EOF", Read());
|
||||
@ -409,7 +420,7 @@ TEST_F(LogTest, BadLengthAtEndIsIgnored) {
|
||||
ASSERT_EQ("", ReportMessage());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ChecksumMismatch) {
|
||||
TEST(LogTest, ChecksumMismatch) {
|
||||
Write("foo");
|
||||
IncrementByte(0, 10);
|
||||
ASSERT_EQ("EOF", Read());
|
||||
@ -417,7 +428,7 @@ TEST_F(LogTest, ChecksumMismatch) {
|
||||
ASSERT_EQ("OK", MatchError("checksum mismatch"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, UnexpectedMiddleType) {
|
||||
TEST(LogTest, UnexpectedMiddleType) {
|
||||
Write("foo");
|
||||
SetByte(6, kMiddleType);
|
||||
FixChecksum(0, 3);
|
||||
@ -426,7 +437,7 @@ TEST_F(LogTest, UnexpectedMiddleType) {
|
||||
ASSERT_EQ("OK", MatchError("missing start"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, UnexpectedLastType) {
|
||||
TEST(LogTest, UnexpectedLastType) {
|
||||
Write("foo");
|
||||
SetByte(6, kLastType);
|
||||
FixChecksum(0, 3);
|
||||
@ -435,7 +446,7 @@ TEST_F(LogTest, UnexpectedLastType) {
|
||||
ASSERT_EQ("OK", MatchError("missing start"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, UnexpectedFullType) {
|
||||
TEST(LogTest, UnexpectedFullType) {
|
||||
Write("foo");
|
||||
Write("bar");
|
||||
SetByte(6, kFirstType);
|
||||
@ -446,7 +457,7 @@ TEST_F(LogTest, UnexpectedFullType) {
|
||||
ASSERT_EQ("OK", MatchError("partial record without end"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, UnexpectedFirstType) {
|
||||
TEST(LogTest, UnexpectedFirstType) {
|
||||
Write("foo");
|
||||
Write(BigString("bar", 100000));
|
||||
SetByte(6, kFirstType);
|
||||
@ -457,7 +468,7 @@ TEST_F(LogTest, UnexpectedFirstType) {
|
||||
ASSERT_EQ("OK", MatchError("partial record without end"));
|
||||
}
|
||||
|
||||
TEST_F(LogTest, MissingLastIsIgnored) {
|
||||
TEST(LogTest, MissingLastIsIgnored) {
|
||||
Write(BigString("bar", kBlockSize));
|
||||
// Remove the LAST block, including header.
|
||||
ShrinkSize(14);
|
||||
@ -466,7 +477,7 @@ TEST_F(LogTest, MissingLastIsIgnored) {
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, PartialLastIsIgnored) {
|
||||
TEST(LogTest, PartialLastIsIgnored) {
|
||||
Write(BigString("bar", kBlockSize));
|
||||
// Cause a bad record length in the LAST block.
|
||||
ShrinkSize(1);
|
||||
@ -475,13 +486,13 @@ TEST_F(LogTest, PartialLastIsIgnored) {
|
||||
ASSERT_EQ(0, DroppedBytes());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, SkipIntoMultiRecord) {
|
||||
TEST(LogTest, SkipIntoMultiRecord) {
|
||||
// Consider a fragmented record:
|
||||
// first(R1), middle(R1), last(R1), first(R2)
|
||||
// If initial_offset points to a record after first(R1) but before first(R2)
|
||||
// incomplete fragment errors are not actual errors, and must be suppressed
|
||||
// until a new first or full record is encountered.
|
||||
Write(BigString("foo", 3 * kBlockSize));
|
||||
Write(BigString("foo", 3*kBlockSize));
|
||||
Write("correct");
|
||||
StartReadingAt(kBlockSize);
|
||||
|
||||
@ -491,7 +502,7 @@ TEST_F(LogTest, SkipIntoMultiRecord) {
|
||||
ASSERT_EQ("EOF", Read());
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ErrorJoinsRecords) {
|
||||
TEST(LogTest, ErrorJoinsRecords) {
|
||||
// Consider two fragmented records:
|
||||
// first(R1) last(R1) first(R2) last(R2)
|
||||
// where the middle two fragments disappear. We do not want
|
||||
@ -503,56 +514,78 @@ TEST_F(LogTest, ErrorJoinsRecords) {
|
||||
Write("correct");
|
||||
|
||||
// Wipe the middle block
|
||||
for (int offset = kBlockSize; offset < 2 * kBlockSize; offset++) {
|
||||
for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
|
||||
SetByte(offset, 'x');
|
||||
}
|
||||
|
||||
ASSERT_EQ("correct", Read());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
const size_t dropped = DroppedBytes();
|
||||
ASSERT_LE(dropped, 2 * kBlockSize + 100);
|
||||
ASSERT_GE(dropped, 2 * kBlockSize);
|
||||
ASSERT_LE(dropped, 2*kBlockSize + 100);
|
||||
ASSERT_GE(dropped, 2*kBlockSize);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadStart) { CheckInitialOffsetRecord(0, 0); }
|
||||
TEST(LogTest, ReadStart) {
|
||||
CheckInitialOffsetRecord(0, 0);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadSecondOneOff) { CheckInitialOffsetRecord(1, 1); }
|
||||
TEST(LogTest, ReadSecondOneOff) {
|
||||
CheckInitialOffsetRecord(1, 1);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadSecondTenThousand) { CheckInitialOffsetRecord(10000, 1); }
|
||||
TEST(LogTest, ReadSecondTenThousand) {
|
||||
CheckInitialOffsetRecord(10000, 1);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadSecondStart) { CheckInitialOffsetRecord(10007, 1); }
|
||||
TEST(LogTest, ReadSecondStart) {
|
||||
CheckInitialOffsetRecord(10007, 1);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadThirdOneOff) { CheckInitialOffsetRecord(10008, 2); }
|
||||
TEST(LogTest, ReadThirdOneOff) {
|
||||
CheckInitialOffsetRecord(10008, 2);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadThirdStart) { CheckInitialOffsetRecord(20014, 2); }
|
||||
TEST(LogTest, ReadThirdStart) {
|
||||
CheckInitialOffsetRecord(20014, 2);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadFourthOneOff) { CheckInitialOffsetRecord(20015, 3); }
|
||||
TEST(LogTest, ReadFourthOneOff) {
|
||||
CheckInitialOffsetRecord(20015, 3);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadFourthFirstBlockTrailer) {
|
||||
TEST(LogTest, ReadFourthFirstBlockTrailer) {
|
||||
CheckInitialOffsetRecord(log::kBlockSize - 4, 3);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadFourthMiddleBlock) {
|
||||
TEST(LogTest, ReadFourthMiddleBlock) {
|
||||
CheckInitialOffsetRecord(log::kBlockSize + 1, 3);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadFourthLastBlock) {
|
||||
TEST(LogTest, ReadFourthLastBlock) {
|
||||
CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadFourthStart) {
|
||||
TEST(LogTest, ReadFourthStart) {
|
||||
CheckInitialOffsetRecord(
|
||||
2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
|
||||
3);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadInitialOffsetIntoBlockPadding) {
|
||||
TEST(LogTest, ReadInitialOffsetIntoBlockPadding) {
|
||||
CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); }
|
||||
TEST(LogTest, ReadEnd) {
|
||||
CheckOffsetPastEndReturnsNoRecords(0);
|
||||
}
|
||||
|
||||
TEST_F(LogTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); }
|
||||
TEST(LogTest, ReadPastEnd) {
|
||||
CheckOffsetPastEndReturnsNoRecords(5);
|
||||
}
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -4,8 +4,7 @@
|
||||
|
||||
#include "db/log_writer.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
@ -20,7 +19,9 @@ static void InitTypeCrc(uint32_t* type_crc) {
|
||||
}
|
||||
}
|
||||
|
||||
Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) {
|
||||
Writer::Writer(WritableFile* dest)
|
||||
: dest_(dest),
|
||||
block_offset_(0) {
|
||||
InitTypeCrc(type_crc_);
|
||||
}
|
||||
|
||||
@ -29,7 +30,8 @@ Writer::Writer(WritableFile* dest, uint64_t dest_length)
|
||||
InitTypeCrc(type_crc_);
|
||||
}
|
||||
|
||||
Writer::~Writer() = default;
|
||||
Writer::~Writer() {
|
||||
}
|
||||
|
||||
Status Writer::AddRecord(const Slice& slice) {
|
||||
const char* ptr = slice.data();
|
||||
@ -47,7 +49,7 @@ Status Writer::AddRecord(const Slice& slice) {
|
||||
// Switch to a new block
|
||||
if (leftover > 0) {
|
||||
// Fill the trailer (literal below relies on kHeaderSize being 7)
|
||||
static_assert(kHeaderSize == 7, "");
|
||||
assert(kHeaderSize == 7);
|
||||
dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
|
||||
}
|
||||
block_offset_ = 0;
|
||||
@ -79,31 +81,30 @@ Status Writer::AddRecord(const Slice& slice) {
|
||||
return s;
|
||||
}
|
||||
|
||||
Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr,
|
||||
size_t length) {
|
||||
assert(length <= 0xffff); // Must fit in two bytes
|
||||
assert(block_offset_ + kHeaderSize + length <= kBlockSize);
|
||||
Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
|
||||
assert(n <= 0xffff); // Must fit in two bytes
|
||||
assert(block_offset_ + kHeaderSize + n <= kBlockSize);
|
||||
|
||||
// Format the header
|
||||
char buf[kHeaderSize];
|
||||
buf[4] = static_cast<char>(length & 0xff);
|
||||
buf[5] = static_cast<char>(length >> 8);
|
||||
buf[4] = static_cast<char>(n & 0xff);
|
||||
buf[5] = static_cast<char>(n >> 8);
|
||||
buf[6] = static_cast<char>(t);
|
||||
|
||||
// Compute the crc of the record type and the payload.
|
||||
uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length);
|
||||
crc = crc32c::Mask(crc); // Adjust for storage
|
||||
uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n);
|
||||
crc = crc32c::Mask(crc); // Adjust for storage
|
||||
EncodeFixed32(buf, crc);
|
||||
|
||||
// Write the header and the payload
|
||||
Status s = dest_->Append(Slice(buf, kHeaderSize));
|
||||
if (s.ok()) {
|
||||
s = dest_->Append(Slice(ptr, length));
|
||||
s = dest_->Append(Slice(ptr, n));
|
||||
if (s.ok()) {
|
||||
s = dest_->Flush();
|
||||
}
|
||||
}
|
||||
block_offset_ += kHeaderSize + length;
|
||||
block_offset_ += kHeaderSize + n;
|
||||
return s;
|
||||
}
|
||||
|
||||
|
@ -5,8 +5,7 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "db/log_format.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
@ -29,23 +28,24 @@ class Writer {
|
||||
// "*dest" must remain live while this Writer is in use.
|
||||
Writer(WritableFile* dest, uint64_t dest_length);
|
||||
|
||||
Writer(const Writer&) = delete;
|
||||
Writer& operator=(const Writer&) = delete;
|
||||
|
||||
~Writer();
|
||||
|
||||
Status AddRecord(const Slice& slice);
|
||||
|
||||
private:
|
||||
Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
|
||||
|
||||
WritableFile* dest_;
|
||||
int block_offset_; // Current offset in block
|
||||
int block_offset_; // Current offset in block
|
||||
|
||||
// crc32c values for all supported record types. These are
|
||||
// pre-computed to reduce the overhead of computing the crc of the
|
||||
// record type stored in the header.
|
||||
uint32_t type_crc_[kMaxRecordType + 1];
|
||||
|
||||
Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
|
||||
|
||||
// No copying allowed
|
||||
Writer(const Writer&);
|
||||
void operator=(const Writer&);
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
|
@ -18,15 +18,20 @@ static Slice GetLengthPrefixedSlice(const char* data) {
|
||||
return Slice(p, len);
|
||||
}
|
||||
|
||||
MemTable::MemTable(const InternalKeyComparator& comparator)
|
||||
: comparator_(comparator), refs_(0), table_(comparator_, &arena_) {}
|
||||
MemTable::MemTable(const InternalKeyComparator& cmp)
|
||||
: comparator_(cmp),
|
||||
refs_(0),
|
||||
table_(comparator_, &arena_) {
|
||||
}
|
||||
|
||||
MemTable::~MemTable() { assert(refs_ == 0); }
|
||||
MemTable::~MemTable() {
|
||||
assert(refs_ == 0);
|
||||
}
|
||||
|
||||
size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
|
||||
|
||||
int MemTable::KeyComparator::operator()(const char* aptr,
|
||||
const char* bptr) const {
|
||||
int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
|
||||
const {
|
||||
// Internal keys are encoded as length-prefixed strings.
|
||||
Slice a = GetLengthPrefixedSlice(aptr);
|
||||
Slice b = GetLengthPrefixedSlice(bptr);
|
||||
@ -43,58 +48,59 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) {
|
||||
return scratch->data();
|
||||
}
|
||||
|
||||
class MemTableIterator : public Iterator {
|
||||
class MemTableIterator: public Iterator {
|
||||
public:
|
||||
explicit MemTableIterator(MemTable::Table* table) : iter_(table) {}
|
||||
explicit MemTableIterator(MemTable::Table* table) : iter_(table) { }
|
||||
|
||||
MemTableIterator(const MemTableIterator&) = delete;
|
||||
MemTableIterator& operator=(const MemTableIterator&) = delete;
|
||||
|
||||
~MemTableIterator() override = default;
|
||||
|
||||
bool Valid() const override { return iter_.Valid(); }
|
||||
void Seek(const Slice& k) override { iter_.Seek(EncodeKey(&tmp_, k)); }
|
||||
void SeekToFirst() override { iter_.SeekToFirst(); }
|
||||
void SeekToLast() override { iter_.SeekToLast(); }
|
||||
void Next() override { iter_.Next(); }
|
||||
void Prev() override { iter_.Prev(); }
|
||||
Slice key() const override { return GetLengthPrefixedSlice(iter_.key()); }
|
||||
Slice value() const override {
|
||||
virtual bool Valid() const { return iter_.Valid(); }
|
||||
virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
|
||||
virtual void SeekToFirst() { iter_.SeekToFirst(); }
|
||||
virtual void SeekToLast() { iter_.SeekToLast(); }
|
||||
virtual void Next() { iter_.Next(); }
|
||||
virtual void Prev() { iter_.Prev(); }
|
||||
virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
|
||||
virtual Slice value() const {
|
||||
Slice key_slice = GetLengthPrefixedSlice(iter_.key());
|
||||
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
|
||||
}
|
||||
|
||||
Status status() const override { return Status::OK(); }
|
||||
virtual Status status() const { return Status::OK(); }
|
||||
|
||||
private:
|
||||
MemTable::Table::Iterator iter_;
|
||||
std::string tmp_; // For passing to EncodeKey
|
||||
std::string tmp_; // For passing to EncodeKey
|
||||
|
||||
// No copying allowed
|
||||
MemTableIterator(const MemTableIterator&);
|
||||
void operator=(const MemTableIterator&);
|
||||
};
|
||||
|
||||
Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); }
|
||||
Iterator* MemTable::NewIterator() {
|
||||
return new MemTableIterator(&table_);
|
||||
}
|
||||
|
||||
void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key,
|
||||
void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
const Slice& key,
|
||||
const Slice& value) {
|
||||
// Format of an entry is concatenation of:
|
||||
// key_size : varint32 of internal_key.size()
|
||||
// key bytes : char[internal_key.size()]
|
||||
// tag : uint64((sequence << 8) | type)
|
||||
// value_size : varint32 of value.size()
|
||||
// value bytes : char[value.size()]
|
||||
size_t key_size = key.size();
|
||||
size_t val_size = value.size();
|
||||
size_t internal_key_size = key_size + 8;
|
||||
const size_t encoded_len = VarintLength(internal_key_size) +
|
||||
internal_key_size + VarintLength(val_size) +
|
||||
val_size;
|
||||
const size_t encoded_len =
|
||||
VarintLength(internal_key_size) + internal_key_size +
|
||||
VarintLength(val_size) + val_size;
|
||||
char* buf = arena_.Allocate(encoded_len);
|
||||
char* p = EncodeVarint32(buf, internal_key_size);
|
||||
std::memcpy(p, key.data(), key_size);
|
||||
memcpy(p, key.data(), key_size);
|
||||
p += key_size;
|
||||
EncodeFixed64(p, (s << 8) | type);
|
||||
p += 8;
|
||||
p = EncodeVarint32(p, val_size);
|
||||
std::memcpy(p, value.data(), val_size);
|
||||
memcpy(p, value.data(), val_size);
|
||||
assert(p + val_size == buf + encoded_len);
|
||||
table_.Insert(buf);
|
||||
}
|
||||
@ -115,9 +121,10 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
|
||||
// all entries with overly large sequence numbers.
|
||||
const char* entry = iter.key();
|
||||
uint32_t key_length;
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
|
||||
if (comparator_.comparator.user_comparator()->Compare(
|
||||
Slice(key_ptr, key_length - 8), key.user_key()) == 0) {
|
||||
Slice(key_ptr, key_length - 8),
|
||||
key.user_key()) == 0) {
|
||||
// Correct user key
|
||||
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
|
||||
switch (static_cast<ValueType>(tag & 0xff)) {
|
||||
|
@ -6,10 +6,9 @@
|
||||
#define STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/skiplist.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "util/arena.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -23,9 +22,6 @@ class MemTable {
|
||||
// is zero and the caller must call Ref() at least once.
|
||||
explicit MemTable(const InternalKeyComparator& comparator);
|
||||
|
||||
MemTable(const MemTable&) = delete;
|
||||
MemTable& operator=(const MemTable&) = delete;
|
||||
|
||||
// Increase reference count.
|
||||
void Ref() { ++refs_; }
|
||||
|
||||
@ -53,7 +49,8 @@ class MemTable {
|
||||
// Add an entry into memtable that maps key to value at the
|
||||
// specified sequence number and with the specified type.
|
||||
// Typically value will be empty if type==kTypeDeletion.
|
||||
void Add(SequenceNumber seq, ValueType type, const Slice& key,
|
||||
void Add(SequenceNumber seq, ValueType type,
|
||||
const Slice& key,
|
||||
const Slice& value);
|
||||
|
||||
// If memtable contains a value for key, store it in *value and return true.
|
||||
@ -63,23 +60,26 @@ class MemTable {
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s);
|
||||
|
||||
private:
|
||||
friend class MemTableIterator;
|
||||
friend class MemTableBackwardIterator;
|
||||
~MemTable(); // Private since only Unref() should be used to delete it
|
||||
|
||||
struct KeyComparator {
|
||||
const InternalKeyComparator comparator;
|
||||
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
|
||||
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
|
||||
int operator()(const char* a, const char* b) const;
|
||||
};
|
||||
friend class MemTableIterator;
|
||||
friend class MemTableBackwardIterator;
|
||||
|
||||
typedef SkipList<const char*, KeyComparator> Table;
|
||||
|
||||
~MemTable(); // Private since only Unref() should be used to delete it
|
||||
|
||||
KeyComparator comparator_;
|
||||
int refs_;
|
||||
Arena arena_;
|
||||
Table table_;
|
||||
|
||||
// No copying allowed
|
||||
MemTable(const MemTable&);
|
||||
void operator=(const MemTable&);
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -2,7 +2,6 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/version_set.h"
|
||||
@ -11,14 +10,15 @@
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class RecoveryTest : public testing::Test {
|
||||
class RecoveryTest {
|
||||
public:
|
||||
RecoveryTest() : env_(Env::Default()), db_(nullptr) {
|
||||
dbname_ = testing::TempDir() + "recovery_test";
|
||||
dbname_ = test::TmpDir() + "/recovery_test";
|
||||
DestroyDB(dbname_, Options());
|
||||
Open();
|
||||
}
|
||||
@ -63,7 +63,7 @@ class RecoveryTest : public testing::Test {
|
||||
}
|
||||
|
||||
void Open(Options* options = nullptr) {
|
||||
ASSERT_LEVELDB_OK(OpenWithStatus(options));
|
||||
ASSERT_OK(OpenWithStatus(options));
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
}
|
||||
|
||||
@ -84,37 +84,40 @@ class RecoveryTest : public testing::Test {
|
||||
|
||||
std::string ManifestFileName() {
|
||||
std::string current;
|
||||
EXPECT_LEVELDB_OK(
|
||||
ReadFileToString(env_, CurrentFileName(dbname_), ¤t));
|
||||
ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t));
|
||||
size_t len = current.size();
|
||||
if (len > 0 && current[len - 1] == '\n') {
|
||||
if (len > 0 && current[len-1] == '\n') {
|
||||
current.resize(len - 1);
|
||||
}
|
||||
return dbname_ + "/" + current;
|
||||
}
|
||||
|
||||
std::string LogName(uint64_t number) { return LogFileName(dbname_, number); }
|
||||
std::string LogName(uint64_t number) {
|
||||
return LogFileName(dbname_, number);
|
||||
}
|
||||
|
||||
size_t RemoveLogFiles() {
|
||||
size_t DeleteLogFiles() {
|
||||
// Linux allows unlinking open files, but Windows does not.
|
||||
// Closing the db allows for file deletion.
|
||||
Close();
|
||||
std::vector<uint64_t> logs = GetFiles(kLogFile);
|
||||
for (size_t i = 0; i < logs.size(); i++) {
|
||||
EXPECT_LEVELDB_OK(env_->RemoveFile(LogName(logs[i]))) << LogName(logs[i]);
|
||||
ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
|
||||
}
|
||||
return logs.size();
|
||||
}
|
||||
|
||||
void RemoveManifestFile() {
|
||||
ASSERT_LEVELDB_OK(env_->RemoveFile(ManifestFileName()));
|
||||
void DeleteManifestFile() {
|
||||
ASSERT_OK(env_->DeleteFile(ManifestFileName()));
|
||||
}
|
||||
|
||||
uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; }
|
||||
uint64_t FirstLogFile() {
|
||||
return GetFiles(kLogFile)[0];
|
||||
}
|
||||
|
||||
std::vector<uint64_t> GetFiles(FileType t) {
|
||||
std::vector<std::string> filenames;
|
||||
EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames));
|
||||
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
|
||||
std::vector<uint64_t> result;
|
||||
for (size_t i = 0; i < filenames.size(); i++) {
|
||||
uint64_t number;
|
||||
@ -126,29 +129,35 @@ class RecoveryTest : public testing::Test {
|
||||
return result;
|
||||
}
|
||||
|
||||
int NumLogs() { return GetFiles(kLogFile).size(); }
|
||||
int NumLogs() {
|
||||
return GetFiles(kLogFile).size();
|
||||
}
|
||||
|
||||
int NumTables() { return GetFiles(kTableFile).size(); }
|
||||
int NumTables() {
|
||||
return GetFiles(kTableFile).size();
|
||||
}
|
||||
|
||||
uint64_t FileSize(const std::string& fname) {
|
||||
uint64_t result;
|
||||
EXPECT_LEVELDB_OK(env_->GetFileSize(fname, &result)) << fname;
|
||||
ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
|
||||
return result;
|
||||
}
|
||||
|
||||
void CompactMemTable() { dbfull()->TEST_CompactMemTable(); }
|
||||
void CompactMemTable() {
|
||||
dbfull()->TEST_CompactMemTable();
|
||||
}
|
||||
|
||||
// Directly construct a log file that sets key to val.
|
||||
void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
|
||||
std::string fname = LogFileName(dbname_, lognum);
|
||||
WritableFile* file;
|
||||
ASSERT_LEVELDB_OK(env_->NewWritableFile(fname, &file));
|
||||
ASSERT_OK(env_->NewWritableFile(fname, &file));
|
||||
log::Writer writer(file);
|
||||
WriteBatch batch;
|
||||
batch.Put(key, val);
|
||||
WriteBatchInternal::SetSequence(&batch, seq);
|
||||
ASSERT_LEVELDB_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
|
||||
ASSERT_LEVELDB_OK(file->Flush());
|
||||
ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
|
||||
ASSERT_OK(file->Flush());
|
||||
delete file;
|
||||
}
|
||||
|
||||
@ -158,13 +167,12 @@ class RecoveryTest : public testing::Test {
|
||||
DB* db_;
|
||||
};
|
||||
|
||||
TEST_F(RecoveryTest, ManifestReused) {
|
||||
TEST(RecoveryTest, ManifestReused) {
|
||||
if (!CanAppend()) {
|
||||
std::fprintf(stderr,
|
||||
"skipping test because env does not support appending\n");
|
||||
fprintf(stderr, "skipping test because env does not support appending\n");
|
||||
return;
|
||||
}
|
||||
ASSERT_LEVELDB_OK(Put("foo", "bar"));
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
std::string old_manifest = ManifestFileName();
|
||||
Open();
|
||||
@ -175,13 +183,12 @@ TEST_F(RecoveryTest, ManifestReused) {
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
|
||||
TEST_F(RecoveryTest, LargeManifestCompacted) {
|
||||
TEST(RecoveryTest, LargeManifestCompacted) {
|
||||
if (!CanAppend()) {
|
||||
std::fprintf(stderr,
|
||||
"skipping test because env does not support appending\n");
|
||||
fprintf(stderr, "skipping test because env does not support appending\n");
|
||||
return;
|
||||
}
|
||||
ASSERT_LEVELDB_OK(Put("foo", "bar"));
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
std::string old_manifest = ManifestFileName();
|
||||
|
||||
@ -189,10 +196,10 @@ TEST_F(RecoveryTest, LargeManifestCompacted) {
|
||||
{
|
||||
uint64_t len = FileSize(old_manifest);
|
||||
WritableFile* file;
|
||||
ASSERT_LEVELDB_OK(env()->NewAppendableFile(old_manifest, &file));
|
||||
std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0);
|
||||
ASSERT_LEVELDB_OK(file->Append(zeroes));
|
||||
ASSERT_LEVELDB_OK(file->Flush());
|
||||
ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
|
||||
std::string zeroes(3*1048576 - static_cast<size_t>(len), 0);
|
||||
ASSERT_OK(file->Append(zeroes));
|
||||
ASSERT_OK(file->Flush());
|
||||
delete file;
|
||||
}
|
||||
|
||||
@ -207,23 +214,22 @@ TEST_F(RecoveryTest, LargeManifestCompacted) {
|
||||
ASSERT_EQ("bar", Get("foo"));
|
||||
}
|
||||
|
||||
TEST_F(RecoveryTest, NoLogFiles) {
|
||||
ASSERT_LEVELDB_OK(Put("foo", "bar"));
|
||||
ASSERT_EQ(1, RemoveLogFiles());
|
||||
TEST(RecoveryTest, NoLogFiles) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_EQ(1, DeleteLogFiles());
|
||||
Open();
|
||||
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
||||
Open();
|
||||
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
||||
}
|
||||
|
||||
TEST_F(RecoveryTest, LogFileReuse) {
|
||||
TEST(RecoveryTest, LogFileReuse) {
|
||||
if (!CanAppend()) {
|
||||
std::fprintf(stderr,
|
||||
"skipping test because env does not support appending\n");
|
||||
fprintf(stderr, "skipping test because env does not support appending\n");
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < 2; i++) {
|
||||
ASSERT_LEVELDB_OK(Put("foo", "bar"));
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
if (i == 0) {
|
||||
// Compact to ensure current log is empty
|
||||
CompactMemTable();
|
||||
@ -247,13 +253,13 @@ TEST_F(RecoveryTest, LogFileReuse) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(RecoveryTest, MultipleMemTables) {
|
||||
TEST(RecoveryTest, MultipleMemTables) {
|
||||
// Make a large log.
|
||||
const int kNum = 1000;
|
||||
for (int i = 0; i < kNum; i++) {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "%050d", i);
|
||||
ASSERT_LEVELDB_OK(Put(buf, buf));
|
||||
snprintf(buf, sizeof(buf), "%050d", i);
|
||||
ASSERT_OK(Put(buf, buf));
|
||||
}
|
||||
ASSERT_EQ(0, NumTables());
|
||||
Close();
|
||||
@ -264,35 +270,35 @@ TEST_F(RecoveryTest, MultipleMemTables) {
|
||||
// Force creation of multiple memtables by reducing the write buffer size.
|
||||
Options opt;
|
||||
opt.reuse_logs = true;
|
||||
opt.write_buffer_size = (kNum * 100) / 2;
|
||||
opt.write_buffer_size = (kNum*100) / 2;
|
||||
Open(&opt);
|
||||
ASSERT_LE(2, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
|
||||
for (int i = 0; i < kNum; i++) {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "%050d", i);
|
||||
snprintf(buf, sizeof(buf), "%050d", i);
|
||||
ASSERT_EQ(buf, Get(buf));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(RecoveryTest, MultipleLogFiles) {
|
||||
ASSERT_LEVELDB_OK(Put("foo", "bar"));
|
||||
TEST(RecoveryTest, MultipleLogFiles) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
|
||||
// Make a bunch of uncompacted log files.
|
||||
uint64_t old_log = FirstLogFile();
|
||||
MakeLogFile(old_log + 1, 1000, "hello", "world");
|
||||
MakeLogFile(old_log + 2, 1001, "hi", "there");
|
||||
MakeLogFile(old_log + 3, 1002, "foo", "bar2");
|
||||
MakeLogFile(old_log+1, 1000, "hello", "world");
|
||||
MakeLogFile(old_log+2, 1001, "hi", "there");
|
||||
MakeLogFile(old_log+3, 1002, "foo", "bar2");
|
||||
|
||||
// Recover and check that all log files were processed.
|
||||
Open();
|
||||
ASSERT_LE(1, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
uint64_t new_log = FirstLogFile();
|
||||
ASSERT_LE(old_log + 3, new_log);
|
||||
ASSERT_LE(old_log+3, new_log);
|
||||
ASSERT_EQ("bar2", Get("foo"));
|
||||
ASSERT_EQ("world", Get("hello"));
|
||||
ASSERT_EQ("there", Get("hi"));
|
||||
@ -310,7 +316,7 @@ TEST_F(RecoveryTest, MultipleLogFiles) {
|
||||
|
||||
// Check that introducing an older log file does not cause it to be re-read.
|
||||
Close();
|
||||
MakeLogFile(old_log + 1, 2000, "hello", "stale write");
|
||||
MakeLogFile(old_log+1, 2000, "hello", "stale write");
|
||||
Open();
|
||||
ASSERT_LE(1, NumTables());
|
||||
ASSERT_EQ(1, NumLogs());
|
||||
@ -322,18 +328,17 @@ TEST_F(RecoveryTest, MultipleLogFiles) {
|
||||
ASSERT_EQ("there", Get("hi"));
|
||||
}
|
||||
|
||||
TEST_F(RecoveryTest, ManifestMissing) {
|
||||
ASSERT_LEVELDB_OK(Put("foo", "bar"));
|
||||
TEST(RecoveryTest, ManifestMissing) {
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
Close();
|
||||
RemoveManifestFile();
|
||||
DeleteManifestFile();
|
||||
|
||||
Status status = OpenWithStatus();
|
||||
#if defined(LEVELDB_PLATFORM_CHROMIUM)
|
||||
// TODO(crbug.com/760362): See comment in MakeIOError() from env_chromium.cc.
|
||||
ASSERT_TRUE(status.IsIOError());
|
||||
#else
|
||||
ASSERT_TRUE(status.IsCorruption());
|
||||
#endif // defined(LEVELDB_PLATFORM_CHROMIUM)
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
88
db/repair.cc
88
db/repair.cc
@ -84,7 +84,9 @@ class Repairer {
|
||||
"recovered %d files; %llu bytes. "
|
||||
"Some data may have been lost. "
|
||||
"****",
|
||||
dbname_.c_str(), static_cast<int>(tables_.size()), bytes);
|
||||
dbname_.c_str(),
|
||||
static_cast<int>(tables_.size()),
|
||||
bytes);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
@ -95,6 +97,22 @@ class Repairer {
|
||||
SequenceNumber max_sequence;
|
||||
};
|
||||
|
||||
std::string const dbname_;
|
||||
Env* const env_;
|
||||
InternalKeyComparator const icmp_;
|
||||
InternalFilterPolicy const ipolicy_;
|
||||
Options const options_;
|
||||
bool owns_info_log_;
|
||||
bool owns_cache_;
|
||||
TableCache* table_cache_;
|
||||
VersionEdit edit_;
|
||||
|
||||
std::vector<std::string> manifests_;
|
||||
std::vector<uint64_t> table_numbers_;
|
||||
std::vector<uint64_t> logs_;
|
||||
std::vector<TableInfo> tables_;
|
||||
uint64_t next_file_number_;
|
||||
|
||||
Status FindFiles() {
|
||||
std::vector<std::string> filenames;
|
||||
Status status = env_->GetChildren(dbname_, &filenames);
|
||||
@ -134,7 +152,8 @@ class Repairer {
|
||||
Status status = ConvertLogToTable(logs_[i]);
|
||||
if (!status.ok()) {
|
||||
Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
|
||||
(unsigned long long)logs_[i], status.ToString().c_str());
|
||||
(unsigned long long) logs_[i],
|
||||
status.ToString().c_str());
|
||||
}
|
||||
ArchiveFile(logname);
|
||||
}
|
||||
@ -145,10 +164,11 @@ class Repairer {
|
||||
Env* env;
|
||||
Logger* info_log;
|
||||
uint64_t lognum;
|
||||
void Corruption(size_t bytes, const Status& s) override {
|
||||
virtual void Corruption(size_t bytes, const Status& s) {
|
||||
// We print error messages for corruption, but continue repairing.
|
||||
Log(info_log, "Log #%llu: dropping %d bytes; %s",
|
||||
(unsigned long long)lognum, static_cast<int>(bytes),
|
||||
(unsigned long long) lognum,
|
||||
static_cast<int>(bytes),
|
||||
s.ToString().c_str());
|
||||
}
|
||||
};
|
||||
@ -170,8 +190,8 @@ class Repairer {
|
||||
// corruptions cause entire commits to be skipped instead of
|
||||
// propagating bad information (like overly large sequence
|
||||
// numbers).
|
||||
log::Reader reader(lfile, &reporter, false /*do not checksum*/,
|
||||
0 /*initial_offset*/);
|
||||
log::Reader reader(lfile, &reporter, false/*do not checksum*/,
|
||||
0/*initial_offset*/);
|
||||
|
||||
// Read all the records and add to a memtable
|
||||
std::string scratch;
|
||||
@ -182,8 +202,8 @@ class Repairer {
|
||||
int counter = 0;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(record.size(),
|
||||
Status::Corruption("log record too small"));
|
||||
reporter.Corruption(
|
||||
record.size(), Status::Corruption("log record too small"));
|
||||
continue;
|
||||
}
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
@ -192,7 +212,8 @@ class Repairer {
|
||||
counter += WriteBatchInternal::Count(&batch);
|
||||
} else {
|
||||
Log(options_.info_log, "Log #%llu: ignoring %s",
|
||||
(unsigned long long)log, status.ToString().c_str());
|
||||
(unsigned long long) log,
|
||||
status.ToString().c_str());
|
||||
status = Status::OK(); // Keep going with rest of file
|
||||
}
|
||||
}
|
||||
@ -213,7 +234,9 @@ class Repairer {
|
||||
}
|
||||
}
|
||||
Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
|
||||
(unsigned long long)log, counter, (unsigned long long)meta.number,
|
||||
(unsigned long long) log,
|
||||
counter,
|
||||
(unsigned long long) meta.number,
|
||||
status.ToString().c_str());
|
||||
return status;
|
||||
}
|
||||
@ -249,7 +272,8 @@ class Repairer {
|
||||
ArchiveFile(TableFileName(dbname_, number));
|
||||
ArchiveFile(SSTTableFileName(dbname_, number));
|
||||
Log(options_.info_log, "Table #%llu: dropped: %s",
|
||||
(unsigned long long)t.meta.number, status.ToString().c_str());
|
||||
(unsigned long long) t.meta.number,
|
||||
status.ToString().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@ -263,7 +287,8 @@ class Repairer {
|
||||
Slice key = iter->key();
|
||||
if (!ParseInternalKey(key, &parsed)) {
|
||||
Log(options_.info_log, "Table #%llu: unparsable key %s",
|
||||
(unsigned long long)t.meta.number, EscapeString(key).c_str());
|
||||
(unsigned long long) t.meta.number,
|
||||
EscapeString(key).c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -282,7 +307,9 @@ class Repairer {
|
||||
}
|
||||
delete iter;
|
||||
Log(options_.info_log, "Table #%llu: %d entries %s",
|
||||
(unsigned long long)t.meta.number, counter, status.ToString().c_str());
|
||||
(unsigned long long) t.meta.number,
|
||||
counter,
|
||||
status.ToString().c_str());
|
||||
|
||||
if (status.ok()) {
|
||||
tables_.push_back(t);
|
||||
@ -336,12 +363,12 @@ class Repairer {
|
||||
s = env_->RenameFile(copy, orig);
|
||||
if (s.ok()) {
|
||||
Log(options_.info_log, "Table #%llu: %d entries repaired",
|
||||
(unsigned long long)t.meta.number, counter);
|
||||
(unsigned long long) t.meta.number, counter);
|
||||
tables_.push_back(t);
|
||||
}
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env_->RemoveFile(copy);
|
||||
env_->DeleteFile(copy);
|
||||
}
|
||||
}
|
||||
|
||||
@ -368,12 +395,11 @@ class Repairer {
|
||||
for (size_t i = 0; i < tables_.size(); i++) {
|
||||
// TODO(opt): separate out into multiple levels
|
||||
const TableInfo& t = tables_[i];
|
||||
edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest,
|
||||
t.meta.largest);
|
||||
edit_.AddFile(0, t.meta.number, t.meta.file_size,
|
||||
t.meta.smallest, t.meta.largest);
|
||||
}
|
||||
|
||||
// std::fprintf(stderr,
|
||||
// "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
|
||||
//fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
|
||||
{
|
||||
log::Writer log(file);
|
||||
std::string record;
|
||||
@ -387,7 +413,7 @@ class Repairer {
|
||||
file = nullptr;
|
||||
|
||||
if (!status.ok()) {
|
||||
env_->RemoveFile(tmp);
|
||||
env_->DeleteFile(tmp);
|
||||
} else {
|
||||
// Discard older manifests
|
||||
for (size_t i = 0; i < manifests_.size(); i++) {
|
||||
@ -399,7 +425,7 @@ class Repairer {
|
||||
if (status.ok()) {
|
||||
status = SetCurrentFile(env_, dbname_, 1);
|
||||
} else {
|
||||
env_->RemoveFile(tmp);
|
||||
env_->DeleteFile(tmp);
|
||||
}
|
||||
}
|
||||
return status;
|
||||
@ -421,25 +447,9 @@ class Repairer {
|
||||
new_file.append("/");
|
||||
new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
|
||||
Status s = env_->RenameFile(fname, new_file);
|
||||
Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(),
|
||||
s.ToString().c_str());
|
||||
Log(options_.info_log, "Archiving %s: %s\n",
|
||||
fname.c_str(), s.ToString().c_str());
|
||||
}
|
||||
|
||||
const std::string dbname_;
|
||||
Env* const env_;
|
||||
InternalKeyComparator const icmp_;
|
||||
InternalFilterPolicy const ipolicy_;
|
||||
const Options options_;
|
||||
bool owns_info_log_;
|
||||
bool owns_cache_;
|
||||
TableCache* table_cache_;
|
||||
VersionEdit edit_;
|
||||
|
||||
std::vector<std::string> manifests_;
|
||||
std::vector<uint64_t> table_numbers_;
|
||||
std::vector<uint64_t> logs_;
|
||||
std::vector<TableInfo> tables_;
|
||||
uint64_t next_file_number_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
|
@ -36,7 +36,9 @@
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
class Arena;
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
class SkipList {
|
||||
private:
|
||||
struct Node;
|
||||
@ -47,9 +49,6 @@ class SkipList {
|
||||
// must remain allocated for the lifetime of the skiplist object.
|
||||
explicit SkipList(Comparator cmp, Arena* arena);
|
||||
|
||||
SkipList(const SkipList&) = delete;
|
||||
SkipList& operator=(const SkipList&) = delete;
|
||||
|
||||
// Insert key into the list.
|
||||
// REQUIRES: nothing that compares equal to key is currently in the list.
|
||||
void Insert(const Key& key);
|
||||
@ -99,10 +98,23 @@ class SkipList {
|
||||
private:
|
||||
enum { kMaxHeight = 12 };
|
||||
|
||||
// Immutable after construction
|
||||
Comparator const compare_;
|
||||
Arena* const arena_; // Arena used for allocations of nodes
|
||||
|
||||
Node* const head_;
|
||||
|
||||
// Modified only by Insert(). Read racily by readers, but stale
|
||||
// values are ok.
|
||||
std::atomic<int> max_height_; // Height of the entire list
|
||||
|
||||
inline int GetMaxHeight() const {
|
||||
return max_height_.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
// Read/written only by Insert().
|
||||
Random rnd_;
|
||||
|
||||
Node* NewNode(const Key& key, int height);
|
||||
int RandomHeight();
|
||||
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
|
||||
@ -125,24 +137,15 @@ class SkipList {
|
||||
// Return head_ if list is empty.
|
||||
Node* FindLast() const;
|
||||
|
||||
// Immutable after construction
|
||||
Comparator const compare_;
|
||||
Arena* const arena_; // Arena used for allocations of nodes
|
||||
|
||||
Node* const head_;
|
||||
|
||||
// Modified only by Insert(). Read racily by readers, but stale
|
||||
// values are ok.
|
||||
std::atomic<int> max_height_; // Height of the entire list
|
||||
|
||||
// Read/written only by Insert().
|
||||
Random rnd_;
|
||||
// No copying allowed
|
||||
SkipList(const SkipList&);
|
||||
void operator=(const SkipList&);
|
||||
};
|
||||
|
||||
// Implementation details follow
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
struct SkipList<Key, Comparator>::Node {
|
||||
explicit Node(const Key& k) : key(k) {}
|
||||
explicit Node(const Key& k) : key(k) { }
|
||||
|
||||
Key const key;
|
||||
|
||||
@ -176,38 +179,38 @@ struct SkipList<Key, Comparator>::Node {
|
||||
std::atomic<Node*> next_[1];
|
||||
};
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode(
|
||||
const Key& key, int height) {
|
||||
template<typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node*
|
||||
SkipList<Key, Comparator>::NewNode(const Key& key, int height) {
|
||||
char* const node_memory = arena_->AllocateAligned(
|
||||
sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
|
||||
return new (node_memory) Node(key);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline SkipList<Key, Comparator>::Iterator::Iterator(const SkipList* list) {
|
||||
list_ = list;
|
||||
node_ = nullptr;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline bool SkipList<Key, Comparator>::Iterator::Valid() const {
|
||||
return node_ != nullptr;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline const Key& SkipList<Key, Comparator>::Iterator::key() const {
|
||||
assert(Valid());
|
||||
return node_->key;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::Next() {
|
||||
assert(Valid());
|
||||
node_ = node_->Next(0);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::Prev() {
|
||||
// Instead of using explicit "prev" links, we just search for the
|
||||
// last node that falls before key.
|
||||
@ -218,17 +221,17 @@ inline void SkipList<Key, Comparator>::Iterator::Prev() {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::Seek(const Key& target) {
|
||||
node_ = list_->FindGreaterOrEqual(target, nullptr);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::SeekToFirst() {
|
||||
node_ = list_->head_->Next(0);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
inline void SkipList<Key, Comparator>::Iterator::SeekToLast() {
|
||||
node_ = list_->FindLast();
|
||||
if (node_ == list_->head_) {
|
||||
@ -236,12 +239,12 @@ inline void SkipList<Key, Comparator>::Iterator::SeekToLast() {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
int SkipList<Key, Comparator>::RandomHeight() {
|
||||
// Increase height with probability 1 in kBranching
|
||||
static const unsigned int kBranching = 4;
|
||||
int height = 1;
|
||||
while (height < kMaxHeight && rnd_.OneIn(kBranching)) {
|
||||
while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) {
|
||||
height++;
|
||||
}
|
||||
assert(height > 0);
|
||||
@ -249,13 +252,13 @@ int SkipList<Key, Comparator>::RandomHeight() {
|
||||
return height;
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
bool SkipList<Key, Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
|
||||
// null n is considered infinite
|
||||
return (n != nullptr) && (compare_(n->key, key) < 0);
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node*
|
||||
SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key,
|
||||
Node** prev) const {
|
||||
@ -278,7 +281,7 @@ SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key,
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node*
|
||||
SkipList<Key, Comparator>::FindLessThan(const Key& key) const {
|
||||
Node* x = head_;
|
||||
@ -299,7 +302,7 @@ SkipList<Key, Comparator>::FindLessThan(const Key& key) const {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
|
||||
const {
|
||||
Node* x = head_;
|
||||
@ -319,7 +322,7 @@ typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
SkipList<Key, Comparator>::SkipList(Comparator cmp, Arena* arena)
|
||||
: compare_(cmp),
|
||||
arena_(arena),
|
||||
@ -331,7 +334,7 @@ SkipList<Key, Comparator>::SkipList(Comparator cmp, Arena* arena)
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
void SkipList<Key, Comparator>::Insert(const Key& key) {
|
||||
// TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
|
||||
// here since Insert() is externally synchronized.
|
||||
@ -365,7 +368,7 @@ void SkipList<Key, Comparator>::Insert(const Key& key) {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Key, class Comparator>
|
||||
template<typename Key, class Comparator>
|
||||
bool SkipList<Key, Comparator>::Contains(const Key& key) const {
|
||||
Node* x = FindGreaterOrEqual(key, nullptr);
|
||||
if (x != nullptr && Equal(key, x->key)) {
|
||||
|
@ -7,14 +7,13 @@
|
||||
#include <atomic>
|
||||
#include <set>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "port/thread_annotations.h"
|
||||
#include "util/arena.h"
|
||||
#include "util/hash.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testutil.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@ -32,6 +31,8 @@ struct Comparator {
|
||||
}
|
||||
};
|
||||
|
||||
class SkipTest { };
|
||||
|
||||
TEST(SkipTest, Empty) {
|
||||
Arena arena;
|
||||
Comparator cmp;
|
||||
@ -116,7 +117,8 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
|
||||
// Compare against model iterator
|
||||
for (std::set<Key>::reverse_iterator model_iter = keys.rbegin();
|
||||
model_iter != keys.rend(); ++model_iter) {
|
||||
model_iter != keys.rend();
|
||||
++model_iter) {
|
||||
ASSERT_TRUE(iter.Valid());
|
||||
ASSERT_EQ(*model_iter, iter.key());
|
||||
iter.Prev();
|
||||
@ -151,19 +153,19 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
// been concurrently added since the iterator started.
|
||||
class ConcurrentTest {
|
||||
private:
|
||||
static constexpr uint32_t K = 4;
|
||||
static const uint32_t K = 4;
|
||||
|
||||
static uint64_t key(Key key) { return (key >> 40); }
|
||||
static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; }
|
||||
static uint64_t hash(Key key) { return key & 0xff; }
|
||||
|
||||
static uint64_t HashNumbers(uint64_t k, uint64_t g) {
|
||||
uint64_t data[2] = {k, g};
|
||||
uint64_t data[2] = { k, g };
|
||||
return Hash(reinterpret_cast<char*>(data), sizeof(data), 0);
|
||||
}
|
||||
|
||||
static Key MakeKey(uint64_t k, uint64_t g) {
|
||||
static_assert(sizeof(Key) == sizeof(uint64_t), "");
|
||||
assert(sizeof(Key) == sizeof(uint64_t));
|
||||
assert(k <= K); // We sometimes pass K to seek to the end of the skiplist
|
||||
assert(g <= 0xffffffffu);
|
||||
return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff));
|
||||
@ -193,7 +195,9 @@ class ConcurrentTest {
|
||||
void Set(int k, int v) {
|
||||
generation[k].store(v, std::memory_order_release);
|
||||
}
|
||||
int Get(int k) { return generation[k].load(std::memory_order_acquire); }
|
||||
int Get(int k) {
|
||||
return generation[k].load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
State() {
|
||||
for (int k = 0; k < K; k++) {
|
||||
@ -212,7 +216,7 @@ class ConcurrentTest {
|
||||
SkipList<Key, Comparator> list_;
|
||||
|
||||
public:
|
||||
ConcurrentTest() : list_(Comparator(), &arena_) {}
|
||||
ConcurrentTest() : list_(Comparator(), &arena_) { }
|
||||
|
||||
// REQUIRES: External synchronization
|
||||
void WriteStep(Random* rnd) {
|
||||
@ -251,9 +255,11 @@ class ConcurrentTest {
|
||||
// Note that generation 0 is never inserted, so it is ok if
|
||||
// <*,0,*> is missing.
|
||||
ASSERT_TRUE((gen(pos) == 0) ||
|
||||
(gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))))
|
||||
<< "key: " << key(pos) << "; gen: " << gen(pos)
|
||||
<< "; initgen: " << initial_state.Get(key(pos));
|
||||
(gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))
|
||||
) << "key: " << key(pos)
|
||||
<< "; gen: " << gen(pos)
|
||||
<< "; initgen: "
|
||||
<< initial_state.Get(key(pos));
|
||||
|
||||
// Advance to next key in the valid key space
|
||||
if (key(pos) < key(current)) {
|
||||
@ -280,9 +286,7 @@ class ConcurrentTest {
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Needed when building in C++11 mode.
|
||||
constexpr uint32_t ConcurrentTest::K;
|
||||
const uint32_t ConcurrentTest::K;
|
||||
|
||||
// Simple test that does single-threaded testing of the ConcurrentTest
|
||||
// scaffolding.
|
||||
@ -301,10 +305,17 @@ class TestState {
|
||||
int seed_;
|
||||
std::atomic<bool> quit_flag_;
|
||||
|
||||
enum ReaderState { STARTING, RUNNING, DONE };
|
||||
enum ReaderState {
|
||||
STARTING,
|
||||
RUNNING,
|
||||
DONE
|
||||
};
|
||||
|
||||
explicit TestState(int s)
|
||||
: seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {}
|
||||
: seed_(s),
|
||||
quit_flag_(false),
|
||||
state_(STARTING),
|
||||
state_cv_(&mu_) {}
|
||||
|
||||
void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) {
|
||||
mu_.Lock();
|
||||
@ -346,7 +357,7 @@ static void RunConcurrent(int run) {
|
||||
const int kSize = 1000;
|
||||
for (int i = 0; i < N; i++) {
|
||||
if ((i % 100) == 0) {
|
||||
std::fprintf(stderr, "Run %d of %d\n", i, N);
|
||||
fprintf(stderr, "Run %d of %d\n", i, N);
|
||||
}
|
||||
TestState state(seed + 1);
|
||||
Env::Default()->Schedule(ConcurrentReader, &state);
|
||||
@ -366,3 +377,7 @@ TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
|
||||
TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -25,7 +25,7 @@ class SnapshotImpl : public Snapshot {
|
||||
friend class SnapshotList;
|
||||
|
||||
// SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList
|
||||
// implementation operates on the next/previous fields directly.
|
||||
// implementation operates on the next/previous fields direcly.
|
||||
SnapshotImpl* prev_;
|
||||
SnapshotImpl* next_;
|
||||
|
||||
@ -44,14 +44,8 @@ class SnapshotList {
|
||||
}
|
||||
|
||||
bool empty() const { return head_.next_ == &head_; }
|
||||
SnapshotImpl* oldest() const {
|
||||
assert(!empty());
|
||||
return head_.next_;
|
||||
}
|
||||
SnapshotImpl* newest() const {
|
||||
assert(!empty());
|
||||
return head_.prev_;
|
||||
}
|
||||
SnapshotImpl* oldest() const { assert(!empty()); return head_.next_; }
|
||||
SnapshotImpl* newest() const { assert(!empty()); return head_.prev_; }
|
||||
|
||||
// Creates a SnapshotImpl and appends it to the end of the list.
|
||||
SnapshotImpl* New(SequenceNumber sequence_number) {
|
||||
|
@ -29,14 +29,18 @@ static void UnrefEntry(void* arg1, void* arg2) {
|
||||
cache->Release(h);
|
||||
}
|
||||
|
||||
TableCache::TableCache(const std::string& dbname, const Options& options,
|
||||
TableCache::TableCache(const std::string& dbname,
|
||||
const Options& options,
|
||||
int entries)
|
||||
: env_(options.env),
|
||||
dbname_(dbname),
|
||||
options_(options),
|
||||
cache_(NewLRUCache(entries)) {}
|
||||
cache_(NewLRUCache(entries)) {
|
||||
}
|
||||
|
||||
TableCache::~TableCache() { delete cache_; }
|
||||
TableCache::~TableCache() {
|
||||
delete cache_;
|
||||
}
|
||||
|
||||
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
|
||||
Cache::Handle** handle) {
|
||||
@ -76,7 +80,8 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
|
||||
}
|
||||
|
||||
Iterator* TableCache::NewIterator(const ReadOptions& options,
|
||||
uint64_t file_number, uint64_t file_size,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
Table** tableptr) {
|
||||
if (tableptr != nullptr) {
|
||||
*tableptr = nullptr;
|
||||
@ -97,15 +102,17 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
|
||||
return result;
|
||||
}
|
||||
|
||||
Status TableCache::Get(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, const Slice& k, void* arg,
|
||||
void (*handle_result)(void*, const Slice&,
|
||||
const Slice&)) {
|
||||
Status TableCache::Get(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
const Slice& k,
|
||||
void* arg,
|
||||
void (*saver)(void*, const Slice&, const Slice&)) {
|
||||
Cache::Handle* handle = nullptr;
|
||||
Status s = FindTable(file_number, file_size, &handle);
|
||||
if (s.ok()) {
|
||||
Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
|
||||
s = t->InternalGet(options, k, arg, handle_result);
|
||||
s = t->InternalGet(options, k, arg, saver);
|
||||
cache_->Release(handle);
|
||||
}
|
||||
return s;
|
||||
|
@ -7,9 +7,8 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "db/dbformat.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/table.h"
|
||||
@ -22,10 +21,6 @@ class Env;
|
||||
class TableCache {
|
||||
public:
|
||||
TableCache(const std::string& dbname, const Options& options, int entries);
|
||||
|
||||
TableCache(const TableCache&) = delete;
|
||||
TableCache& operator=(const TableCache&) = delete;
|
||||
|
||||
~TableCache();
|
||||
|
||||
// Return an iterator for the specified file number (the corresponding
|
||||
@ -35,25 +30,30 @@ class TableCache {
|
||||
// underlies the returned iterator. The returned "*tableptr" object is owned
|
||||
// by the cache and should not be deleted, and is valid for as long as the
|
||||
// returned iterator is live.
|
||||
Iterator* NewIterator(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, Table** tableptr = nullptr);
|
||||
Iterator* NewIterator(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
Table** tableptr = nullptr);
|
||||
|
||||
// If a seek to internal key "k" in specified file finds an entry,
|
||||
// call (*handle_result)(arg, found_key, found_value).
|
||||
Status Get(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, const Slice& k, void* arg,
|
||||
Status Get(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
const Slice& k,
|
||||
void* arg,
|
||||
void (*handle_result)(void*, const Slice&, const Slice&));
|
||||
|
||||
// Evict any entry for the specified file number
|
||||
void Evict(uint64_t file_number);
|
||||
|
||||
private:
|
||||
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
|
||||
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
const Options& options_;
|
||||
Cache* cache_;
|
||||
|
||||
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -12,15 +12,15 @@ namespace leveldb {
|
||||
// Tag numbers for serialized VersionEdit. These numbers are written to
|
||||
// disk and should not be changed.
|
||||
enum Tag {
|
||||
kComparator = 1,
|
||||
kLogNumber = 2,
|
||||
kNextFileNumber = 3,
|
||||
kLastSequence = 4,
|
||||
kCompactPointer = 5,
|
||||
kDeletedFile = 6,
|
||||
kNewFile = 7,
|
||||
kComparator = 1,
|
||||
kLogNumber = 2,
|
||||
kNextFileNumber = 3,
|
||||
kLastSequence = 4,
|
||||
kCompactPointer = 5,
|
||||
kDeletedFile = 6,
|
||||
kNewFile = 7,
|
||||
// 8 was used for large value refs
|
||||
kPrevLogNumber = 9
|
||||
kPrevLogNumber = 9
|
||||
};
|
||||
|
||||
void VersionEdit::Clear() {
|
||||
@ -34,7 +34,6 @@ void VersionEdit::Clear() {
|
||||
has_prev_log_number_ = false;
|
||||
has_next_file_number_ = false;
|
||||
has_last_sequence_ = false;
|
||||
compact_pointers_.clear();
|
||||
deleted_files_.clear();
|
||||
new_files_.clear();
|
||||
}
|
||||
@ -67,10 +66,12 @@ void VersionEdit::EncodeTo(std::string* dst) const {
|
||||
PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode());
|
||||
}
|
||||
|
||||
for (const auto& deleted_file_kvp : deleted_files_) {
|
||||
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
||||
iter != deleted_files_.end();
|
||||
++iter) {
|
||||
PutVarint32(dst, kDeletedFile);
|
||||
PutVarint32(dst, deleted_file_kvp.first); // level
|
||||
PutVarint64(dst, deleted_file_kvp.second); // file number
|
||||
PutVarint32(dst, iter->first); // level
|
||||
PutVarint64(dst, iter->second); // file number
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < new_files_.size(); i++) {
|
||||
@ -87,7 +88,8 @@ void VersionEdit::EncodeTo(std::string* dst) const {
|
||||
static bool GetInternalKey(Slice* input, InternalKey* dst) {
|
||||
Slice str;
|
||||
if (GetLengthPrefixedSlice(input, &str)) {
|
||||
return dst->DecodeFrom(str);
|
||||
dst->DecodeFrom(str);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
@ -95,7 +97,8 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
|
||||
|
||||
static bool GetLevel(Slice* input, int* level) {
|
||||
uint32_t v;
|
||||
if (GetVarint32(input, &v) && v < config::kNumLevels) {
|
||||
if (GetVarint32(input, &v) &&
|
||||
v < config::kNumLevels) {
|
||||
*level = v;
|
||||
return true;
|
||||
} else {
|
||||
@ -160,7 +163,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
break;
|
||||
|
||||
case kCompactPointer:
|
||||
if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) {
|
||||
if (GetLevel(&input, &level) &&
|
||||
GetInternalKey(&input, &key)) {
|
||||
compact_pointers_.push_back(std::make_pair(level, key));
|
||||
} else {
|
||||
msg = "compaction pointer";
|
||||
@ -168,7 +172,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
break;
|
||||
|
||||
case kDeletedFile:
|
||||
if (GetLevel(&input, &level) && GetVarint64(&input, &number)) {
|
||||
if (GetLevel(&input, &level) &&
|
||||
GetVarint64(&input, &number)) {
|
||||
deleted_files_.insert(std::make_pair(level, number));
|
||||
} else {
|
||||
msg = "deleted file";
|
||||
@ -176,7 +181,8 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
break;
|
||||
|
||||
case kNewFile:
|
||||
if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) &&
|
||||
if (GetLevel(&input, &level) &&
|
||||
GetVarint64(&input, &f.number) &&
|
||||
GetVarint64(&input, &f.file_size) &&
|
||||
GetInternalKey(&input, &f.smallest) &&
|
||||
GetInternalKey(&input, &f.largest)) {
|
||||
@ -232,11 +238,13 @@ std::string VersionEdit::DebugString() const {
|
||||
r.append(" ");
|
||||
r.append(compact_pointers_[i].second.DebugString());
|
||||
}
|
||||
for (const auto& deleted_files_kvp : deleted_files_) {
|
||||
r.append("\n RemoveFile: ");
|
||||
AppendNumberTo(&r, deleted_files_kvp.first);
|
||||
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
||||
iter != deleted_files_.end();
|
||||
++iter) {
|
||||
r.append("\n DeleteFile: ");
|
||||
AppendNumberTo(&r, iter->first);
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, deleted_files_kvp.second);
|
||||
AppendNumberTo(&r, iter->second);
|
||||
}
|
||||
for (size_t i = 0; i < new_files_.size(); i++) {
|
||||
const FileMetaData& f = new_files_[i].second;
|
||||
|
@ -8,7 +8,6 @@
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -16,20 +15,20 @@ namespace leveldb {
|
||||
class VersionSet;
|
||||
|
||||
struct FileMetaData {
|
||||
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {}
|
||||
|
||||
int refs;
|
||||
int allowed_seeks; // Seeks allowed until compaction
|
||||
int allowed_seeks; // Seeks allowed until compaction
|
||||
uint64_t number;
|
||||
uint64_t file_size; // File size in bytes
|
||||
InternalKey smallest; // Smallest internal key served by table
|
||||
InternalKey largest; // Largest internal key served by table
|
||||
uint64_t file_size; // File size in bytes
|
||||
InternalKey smallest; // Smallest internal key served by table
|
||||
InternalKey largest; // Largest internal key served by table
|
||||
|
||||
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
|
||||
};
|
||||
|
||||
class VersionEdit {
|
||||
public:
|
||||
VersionEdit() { Clear(); }
|
||||
~VersionEdit() = default;
|
||||
~VersionEdit() { }
|
||||
|
||||
void Clear();
|
||||
|
||||
@ -60,8 +59,10 @@ class VersionEdit {
|
||||
// Add the specified file at the specified number.
|
||||
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
|
||||
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file
|
||||
void AddFile(int level, uint64_t file, uint64_t file_size,
|
||||
const InternalKey& smallest, const InternalKey& largest) {
|
||||
void AddFile(int level, uint64_t file,
|
||||
uint64_t file_size,
|
||||
const InternalKey& smallest,
|
||||
const InternalKey& largest) {
|
||||
FileMetaData f;
|
||||
f.number = file;
|
||||
f.file_size = file_size;
|
||||
@ -71,7 +72,7 @@ class VersionEdit {
|
||||
}
|
||||
|
||||
// Delete the specified "file" from the specified "level".
|
||||
void RemoveFile(int level, uint64_t file) {
|
||||
void DeleteFile(int level, uint64_t file) {
|
||||
deleted_files_.insert(std::make_pair(level, file));
|
||||
}
|
||||
|
||||
@ -83,7 +84,7 @@ class VersionEdit {
|
||||
private:
|
||||
friend class VersionSet;
|
||||
|
||||
typedef std::set<std::pair<int, uint64_t>> DeletedFileSet;
|
||||
typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
|
||||
|
||||
std::string comparator_;
|
||||
uint64_t log_number_;
|
||||
@ -96,9 +97,9 @@ class VersionEdit {
|
||||
bool has_next_file_number_;
|
||||
bool has_last_sequence_;
|
||||
|
||||
std::vector<std::pair<int, InternalKey>> compact_pointers_;
|
||||
std::vector< std::pair<int, InternalKey> > compact_pointers_;
|
||||
DeletedFileSet deleted_files_;
|
||||
std::vector<std::pair<int, FileMetaData>> new_files_;
|
||||
std::vector< std::pair<int, FileMetaData> > new_files_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -3,8 +3,7 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/version_edit.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@ -18,6 +17,8 @@ static void TestEncodeDecode(const VersionEdit& edit) {
|
||||
ASSERT_EQ(encoded, encoded2);
|
||||
}
|
||||
|
||||
class VersionEditTest { };
|
||||
|
||||
TEST(VersionEditTest, EncodeDecode) {
|
||||
static const uint64_t kBig = 1ull << 50;
|
||||
|
||||
@ -27,7 +28,7 @@ TEST(VersionEditTest, EncodeDecode) {
|
||||
edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
|
||||
InternalKey("foo", kBig + 500 + i, kTypeValue),
|
||||
InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
|
||||
edit.RemoveFile(4, kBig + 700 + i);
|
||||
edit.DeleteFile(4, kBig + 700 + i);
|
||||
edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
|
||||
}
|
||||
|
||||
@ -39,3 +40,7 @@ TEST(VersionEditTest, EncodeDecode) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -5,8 +5,7 @@
|
||||
#include "db/version_set.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
|
||||
#include <stdio.h>
|
||||
#include "db/filename.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/log_writer.h"
|
||||
@ -85,7 +84,8 @@ Version::~Version() {
|
||||
}
|
||||
|
||||
int FindFile(const InternalKeyComparator& icmp,
|
||||
const std::vector<FileMetaData*>& files, const Slice& key) {
|
||||
const std::vector<FileMetaData*>& files,
|
||||
const Slice& key) {
|
||||
uint32_t left = 0;
|
||||
uint32_t right = files.size();
|
||||
while (left < right) {
|
||||
@ -104,25 +104,26 @@ int FindFile(const InternalKeyComparator& icmp,
|
||||
return right;
|
||||
}
|
||||
|
||||
static bool AfterFile(const Comparator* ucmp, const Slice* user_key,
|
||||
const FileMetaData* f) {
|
||||
static bool AfterFile(const Comparator* ucmp,
|
||||
const Slice* user_key, const FileMetaData* f) {
|
||||
// null user_key occurs before all keys and is therefore never after *f
|
||||
return (user_key != nullptr &&
|
||||
ucmp->Compare(*user_key, f->largest.user_key()) > 0);
|
||||
}
|
||||
|
||||
static bool BeforeFile(const Comparator* ucmp, const Slice* user_key,
|
||||
const FileMetaData* f) {
|
||||
static bool BeforeFile(const Comparator* ucmp,
|
||||
const Slice* user_key, const FileMetaData* f) {
|
||||
// null user_key occurs after all keys and is therefore never before *f
|
||||
return (user_key != nullptr &&
|
||||
ucmp->Compare(*user_key, f->smallest.user_key()) < 0);
|
||||
}
|
||||
|
||||
bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
||||
bool disjoint_sorted_files,
|
||||
const std::vector<FileMetaData*>& files,
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key) {
|
||||
bool SomeFileOverlapsRange(
|
||||
const InternalKeyComparator& icmp,
|
||||
bool disjoint_sorted_files,
|
||||
const std::vector<FileMetaData*>& files,
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key) {
|
||||
const Comparator* ucmp = icmp.user_comparator();
|
||||
if (!disjoint_sorted_files) {
|
||||
// Need to check against all files
|
||||
@ -142,8 +143,7 @@ bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
||||
uint32_t index = 0;
|
||||
if (smallest_user_key != nullptr) {
|
||||
// Find the earliest possible internal key for smallest_user_key
|
||||
InternalKey small_key(*smallest_user_key, kMaxSequenceNumber,
|
||||
kValueTypeForSeek);
|
||||
InternalKey small_key(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek);
|
||||
index = FindFile(icmp, files, small_key.Encode());
|
||||
}
|
||||
|
||||
@ -164,21 +164,25 @@ class Version::LevelFileNumIterator : public Iterator {
|
||||
public:
|
||||
LevelFileNumIterator(const InternalKeyComparator& icmp,
|
||||
const std::vector<FileMetaData*>* flist)
|
||||
: icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid
|
||||
: icmp_(icmp),
|
||||
flist_(flist),
|
||||
index_(flist->size()) { // Marks as invalid
|
||||
}
|
||||
bool Valid() const override { return index_ < flist_->size(); }
|
||||
void Seek(const Slice& target) override {
|
||||
virtual bool Valid() const {
|
||||
return index_ < flist_->size();
|
||||
}
|
||||
virtual void Seek(const Slice& target) {
|
||||
index_ = FindFile(icmp_, *flist_, target);
|
||||
}
|
||||
void SeekToFirst() override { index_ = 0; }
|
||||
void SeekToLast() override {
|
||||
virtual void SeekToFirst() { index_ = 0; }
|
||||
virtual void SeekToLast() {
|
||||
index_ = flist_->empty() ? 0 : flist_->size() - 1;
|
||||
}
|
||||
void Next() override {
|
||||
virtual void Next() {
|
||||
assert(Valid());
|
||||
index_++;
|
||||
}
|
||||
void Prev() override {
|
||||
virtual void Prev() {
|
||||
assert(Valid());
|
||||
if (index_ == 0) {
|
||||
index_ = flist_->size(); // Marks as invalid
|
||||
@ -186,18 +190,17 @@ class Version::LevelFileNumIterator : public Iterator {
|
||||
index_--;
|
||||
}
|
||||
}
|
||||
Slice key() const override {
|
||||
Slice key() const {
|
||||
assert(Valid());
|
||||
return (*flist_)[index_]->largest.Encode();
|
||||
}
|
||||
Slice value() const override {
|
||||
Slice value() const {
|
||||
assert(Valid());
|
||||
EncodeFixed64(value_buf_, (*flist_)[index_]->number);
|
||||
EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size);
|
||||
EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
|
||||
return Slice(value_buf_, sizeof(value_buf_));
|
||||
}
|
||||
Status status() const override { return Status::OK(); }
|
||||
|
||||
virtual Status status() const { return Status::OK(); }
|
||||
private:
|
||||
const InternalKeyComparator icmp_;
|
||||
const std::vector<FileMetaData*>* const flist_;
|
||||
@ -207,14 +210,16 @@ class Version::LevelFileNumIterator : public Iterator {
|
||||
mutable char value_buf_[16];
|
||||
};
|
||||
|
||||
static Iterator* GetFileIterator(void* arg, const ReadOptions& options,
|
||||
static Iterator* GetFileIterator(void* arg,
|
||||
const ReadOptions& options,
|
||||
const Slice& file_value) {
|
||||
TableCache* cache = reinterpret_cast<TableCache*>(arg);
|
||||
if (file_value.size() != 16) {
|
||||
return NewErrorIterator(
|
||||
Status::Corruption("FileReader invoked with unexpected value"));
|
||||
} else {
|
||||
return cache->NewIterator(options, DecodeFixed64(file_value.data()),
|
||||
return cache->NewIterator(options,
|
||||
DecodeFixed64(file_value.data()),
|
||||
DecodeFixed64(file_value.data() + 8));
|
||||
}
|
||||
}
|
||||
@ -222,16 +227,17 @@ static Iterator* GetFileIterator(void* arg, const ReadOptions& options,
|
||||
Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
|
||||
int level) const {
|
||||
return NewTwoLevelIterator(
|
||||
new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator,
|
||||
vset_->table_cache_, options);
|
||||
new LevelFileNumIterator(vset_->icmp_, &files_[level]),
|
||||
&GetFileIterator, vset_->table_cache_, options);
|
||||
}
|
||||
|
||||
void Version::AddIterators(const ReadOptions& options,
|
||||
std::vector<Iterator*>* iters) {
|
||||
// Merge all level zero files together since they may overlap
|
||||
for (size_t i = 0; i < files_[0].size(); i++) {
|
||||
iters->push_back(vset_->table_cache_->NewIterator(
|
||||
options, files_[0][i]->number, files_[0][i]->file_size));
|
||||
iters->push_back(
|
||||
vset_->table_cache_->NewIterator(
|
||||
options, files_[0][i]->number, files_[0][i]->file_size));
|
||||
}
|
||||
|
||||
// For levels > 0, we can use a concatenating iterator that sequentially
|
||||
@ -258,7 +264,7 @@ struct Saver {
|
||||
Slice user_key;
|
||||
std::string* value;
|
||||
};
|
||||
} // namespace
|
||||
}
|
||||
static void SaveValue(void* arg, const Slice& ikey, const Slice& v) {
|
||||
Saver* s = reinterpret_cast<Saver*>(arg);
|
||||
ParsedInternalKey parsed_key;
|
||||
@ -278,8 +284,10 @@ static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
|
||||
return a->number > b->number;
|
||||
}
|
||||
|
||||
void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
|
||||
void Version::ForEachOverlapping(Slice user_key, Slice internal_key,
|
||||
void* arg,
|
||||
bool (*func)(void*, int, FileMetaData*)) {
|
||||
// TODO(sanjay): Change Version::Get() to use this function.
|
||||
const Comparator* ucmp = vset_->icmp_.user_comparator();
|
||||
|
||||
// Search level-0 in order from newest to oldest.
|
||||
@ -321,82 +329,103 @@ void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
|
||||
}
|
||||
}
|
||||
|
||||
Status Version::Get(const ReadOptions& options, const LookupKey& k,
|
||||
std::string* value, GetStats* stats) {
|
||||
Status Version::Get(const ReadOptions& options,
|
||||
const LookupKey& k,
|
||||
std::string* value,
|
||||
GetStats* stats) {
|
||||
Slice ikey = k.internal_key();
|
||||
Slice user_key = k.user_key();
|
||||
const Comparator* ucmp = vset_->icmp_.user_comparator();
|
||||
Status s;
|
||||
|
||||
stats->seek_file = nullptr;
|
||||
stats->seek_file_level = -1;
|
||||
FileMetaData* last_file_read = nullptr;
|
||||
int last_file_read_level = -1;
|
||||
|
||||
struct State {
|
||||
Saver saver;
|
||||
GetStats* stats;
|
||||
const ReadOptions* options;
|
||||
Slice ikey;
|
||||
FileMetaData* last_file_read;
|
||||
int last_file_read_level;
|
||||
// We can search level-by-level since entries never hop across
|
||||
// levels. Therefore we are guaranteed that if we find data
|
||||
// in a smaller level, later levels are irrelevant.
|
||||
std::vector<FileMetaData*> tmp;
|
||||
FileMetaData* tmp2;
|
||||
for (int level = 0; level < config::kNumLevels; level++) {
|
||||
size_t num_files = files_[level].size();
|
||||
if (num_files == 0) continue;
|
||||
|
||||
VersionSet* vset;
|
||||
Status s;
|
||||
bool found;
|
||||
|
||||
static bool Match(void* arg, int level, FileMetaData* f) {
|
||||
State* state = reinterpret_cast<State*>(arg);
|
||||
|
||||
if (state->stats->seek_file == nullptr &&
|
||||
state->last_file_read != nullptr) {
|
||||
// We have had more than one seek for this read. Charge the 1st file.
|
||||
state->stats->seek_file = state->last_file_read;
|
||||
state->stats->seek_file_level = state->last_file_read_level;
|
||||
// Get the list of files to search in this level
|
||||
FileMetaData* const* files = &files_[level][0];
|
||||
if (level == 0) {
|
||||
// Level-0 files may overlap each other. Find all files that
|
||||
// overlap user_key and process them in order from newest to oldest.
|
||||
tmp.reserve(num_files);
|
||||
for (uint32_t i = 0; i < num_files; i++) {
|
||||
FileMetaData* f = files[i];
|
||||
if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
|
||||
ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
|
||||
tmp.push_back(f);
|
||||
}
|
||||
}
|
||||
if (tmp.empty()) continue;
|
||||
|
||||
state->last_file_read = f;
|
||||
state->last_file_read_level = level;
|
||||
|
||||
state->s = state->vset->table_cache_->Get(*state->options, f->number,
|
||||
f->file_size, state->ikey,
|
||||
&state->saver, SaveValue);
|
||||
if (!state->s.ok()) {
|
||||
state->found = true;
|
||||
return false;
|
||||
std::sort(tmp.begin(), tmp.end(), NewestFirst);
|
||||
files = &tmp[0];
|
||||
num_files = tmp.size();
|
||||
} else {
|
||||
// Binary search to find earliest index whose largest key >= ikey.
|
||||
uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
|
||||
if (index >= num_files) {
|
||||
files = nullptr;
|
||||
num_files = 0;
|
||||
} else {
|
||||
tmp2 = files[index];
|
||||
if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
|
||||
// All of "tmp2" is past any data for user_key
|
||||
files = nullptr;
|
||||
num_files = 0;
|
||||
} else {
|
||||
files = &tmp2;
|
||||
num_files = 1;
|
||||
}
|
||||
}
|
||||
switch (state->saver.state) {
|
||||
case kNotFound:
|
||||
return true; // Keep searching in other files
|
||||
case kFound:
|
||||
state->found = true;
|
||||
return false;
|
||||
case kDeleted:
|
||||
return false;
|
||||
case kCorrupt:
|
||||
state->s =
|
||||
Status::Corruption("corrupted key for ", state->saver.user_key);
|
||||
state->found = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// Not reached. Added to avoid false compilation warnings of
|
||||
// "control reaches end of non-void function".
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
State state;
|
||||
state.found = false;
|
||||
state.stats = stats;
|
||||
state.last_file_read = nullptr;
|
||||
state.last_file_read_level = -1;
|
||||
for (uint32_t i = 0; i < num_files; ++i) {
|
||||
if (last_file_read != nullptr && stats->seek_file == nullptr) {
|
||||
// We have had more than one seek for this read. Charge the 1st file.
|
||||
stats->seek_file = last_file_read;
|
||||
stats->seek_file_level = last_file_read_level;
|
||||
}
|
||||
|
||||
state.options = &options;
|
||||
state.ikey = k.internal_key();
|
||||
state.vset = vset_;
|
||||
FileMetaData* f = files[i];
|
||||
last_file_read = f;
|
||||
last_file_read_level = level;
|
||||
|
||||
state.saver.state = kNotFound;
|
||||
state.saver.ucmp = vset_->icmp_.user_comparator();
|
||||
state.saver.user_key = k.user_key();
|
||||
state.saver.value = value;
|
||||
Saver saver;
|
||||
saver.state = kNotFound;
|
||||
saver.ucmp = ucmp;
|
||||
saver.user_key = user_key;
|
||||
saver.value = value;
|
||||
s = vset_->table_cache_->Get(options, f->number, f->file_size,
|
||||
ikey, &saver, SaveValue);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
switch (saver.state) {
|
||||
case kNotFound:
|
||||
break; // Keep searching in other files
|
||||
case kFound:
|
||||
return s;
|
||||
case kDeleted:
|
||||
s = Status::NotFound(Slice()); // Use empty error message for speed
|
||||
return s;
|
||||
case kCorrupt:
|
||||
s = Status::Corruption("corrupted key for ", user_key);
|
||||
return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ForEachOverlapping(state.saver.user_key, state.ikey, &state, &State::Match);
|
||||
|
||||
return state.found ? state.s : Status::NotFound(Slice());
|
||||
return Status::NotFound(Slice()); // Use an empty error message for speed
|
||||
}
|
||||
|
||||
bool Version::UpdateStats(const GetStats& stats) {
|
||||
@ -450,7 +479,9 @@ bool Version::RecordReadSample(Slice internal_key) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void Version::Ref() { ++refs_; }
|
||||
void Version::Ref() {
|
||||
++refs_;
|
||||
}
|
||||
|
||||
void Version::Unref() {
|
||||
assert(this != &vset_->dummy_versions_);
|
||||
@ -461,14 +492,16 @@ void Version::Unref() {
|
||||
}
|
||||
}
|
||||
|
||||
bool Version::OverlapInLevel(int level, const Slice* smallest_user_key,
|
||||
bool Version::OverlapInLevel(int level,
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key) {
|
||||
return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
|
||||
smallest_user_key, largest_user_key);
|
||||
}
|
||||
|
||||
int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key,
|
||||
const Slice& largest_user_key) {
|
||||
int Version::PickLevelForMemTableOutput(
|
||||
const Slice& smallest_user_key,
|
||||
const Slice& largest_user_key) {
|
||||
int level = 0;
|
||||
if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) {
|
||||
// Push to next level if there is no overlap in next level,
|
||||
@ -495,9 +528,11 @@ int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key,
|
||||
}
|
||||
|
||||
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
||||
void Version::GetOverlappingInputs(int level, const InternalKey* begin,
|
||||
const InternalKey* end,
|
||||
std::vector<FileMetaData*>* inputs) {
|
||||
void Version::GetOverlappingInputs(
|
||||
int level,
|
||||
const InternalKey* begin,
|
||||
const InternalKey* end,
|
||||
std::vector<FileMetaData*>* inputs) {
|
||||
assert(level >= 0);
|
||||
assert(level < config::kNumLevels);
|
||||
inputs->clear();
|
||||
@ -509,7 +544,7 @@ void Version::GetOverlappingInputs(int level, const InternalKey* begin,
|
||||
user_end = end->user_key();
|
||||
}
|
||||
const Comparator* user_cmp = vset_->icmp_.user_comparator();
|
||||
for (size_t i = 0; i < files_[level].size();) {
|
||||
for (size_t i = 0; i < files_[level].size(); ) {
|
||||
FileMetaData* f = files_[level][i++];
|
||||
const Slice file_start = f->smallest.user_key();
|
||||
const Slice file_limit = f->largest.user_key();
|
||||
@ -526,8 +561,8 @@ void Version::GetOverlappingInputs(int level, const InternalKey* begin,
|
||||
user_begin = file_start;
|
||||
inputs->clear();
|
||||
i = 0;
|
||||
} else if (end != nullptr &&
|
||||
user_cmp->Compare(file_limit, user_end) > 0) {
|
||||
} else if (end != nullptr && user_cmp->Compare(file_limit,
|
||||
user_end) > 0) {
|
||||
user_end = file_limit;
|
||||
inputs->clear();
|
||||
i = 0;
|
||||
@ -595,7 +630,9 @@ class VersionSet::Builder {
|
||||
|
||||
public:
|
||||
// Initialize a builder with the files from *base and other info from *vset
|
||||
Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) {
|
||||
Builder(VersionSet* vset, Version* base)
|
||||
: vset_(vset),
|
||||
base_(base) {
|
||||
base_->Ref();
|
||||
BySmallestKey cmp;
|
||||
cmp.internal_comparator = &vset_->icmp_;
|
||||
@ -609,8 +646,8 @@ class VersionSet::Builder {
|
||||
const FileSet* added = levels_[level].added_files;
|
||||
std::vector<FileMetaData*> to_unref;
|
||||
to_unref.reserve(added->size());
|
||||
for (FileSet::const_iterator it = added->begin(); it != added->end();
|
||||
++it) {
|
||||
for (FileSet::const_iterator it = added->begin();
|
||||
it != added->end(); ++it) {
|
||||
to_unref.push_back(*it);
|
||||
}
|
||||
delete added;
|
||||
@ -626,7 +663,7 @@ class VersionSet::Builder {
|
||||
}
|
||||
|
||||
// Apply all of the edits in *edit to the current state.
|
||||
void Apply(const VersionEdit* edit) {
|
||||
void Apply(VersionEdit* edit) {
|
||||
// Update compaction pointers
|
||||
for (size_t i = 0; i < edit->compact_pointers_.size(); i++) {
|
||||
const int level = edit->compact_pointers_[i].first;
|
||||
@ -635,9 +672,12 @@ class VersionSet::Builder {
|
||||
}
|
||||
|
||||
// Delete files
|
||||
for (const auto& deleted_file_set_kvp : edit->deleted_files_) {
|
||||
const int level = deleted_file_set_kvp.first;
|
||||
const uint64_t number = deleted_file_set_kvp.second;
|
||||
const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
|
||||
for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
|
||||
iter != del.end();
|
||||
++iter) {
|
||||
const int level = iter->first;
|
||||
const uint64_t number = iter->second;
|
||||
levels_[level].deleted_files.insert(number);
|
||||
}
|
||||
|
||||
@ -678,17 +718,20 @@ class VersionSet::Builder {
|
||||
const std::vector<FileMetaData*>& base_files = base_->files_[level];
|
||||
std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
|
||||
std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
|
||||
const FileSet* added_files = levels_[level].added_files;
|
||||
v->files_[level].reserve(base_files.size() + added_files->size());
|
||||
for (const auto& added_file : *added_files) {
|
||||
const FileSet* added = levels_[level].added_files;
|
||||
v->files_[level].reserve(base_files.size() + added->size());
|
||||
for (FileSet::const_iterator added_iter = added->begin();
|
||||
added_iter != added->end();
|
||||
++added_iter) {
|
||||
// Add all smaller files listed in base_
|
||||
for (std::vector<FileMetaData*>::const_iterator bpos =
|
||||
std::upper_bound(base_iter, base_end, added_file, cmp);
|
||||
base_iter != bpos; ++base_iter) {
|
||||
for (std::vector<FileMetaData*>::const_iterator bpos
|
||||
= std::upper_bound(base_iter, base_end, *added_iter, cmp);
|
||||
base_iter != bpos;
|
||||
++base_iter) {
|
||||
MaybeAddFile(v, level, *base_iter);
|
||||
}
|
||||
|
||||
MaybeAddFile(v, level, added_file);
|
||||
MaybeAddFile(v, level, *added_iter);
|
||||
}
|
||||
|
||||
// Add remaining base files
|
||||
@ -700,13 +743,13 @@ class VersionSet::Builder {
|
||||
// Make sure there is no overlap in levels > 0
|
||||
if (level > 0) {
|
||||
for (uint32_t i = 1; i < v->files_[level].size(); i++) {
|
||||
const InternalKey& prev_end = v->files_[level][i - 1]->largest;
|
||||
const InternalKey& prev_end = v->files_[level][i-1]->largest;
|
||||
const InternalKey& this_begin = v->files_[level][i]->smallest;
|
||||
if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
|
||||
std::fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
|
||||
prev_end.DebugString().c_str(),
|
||||
this_begin.DebugString().c_str());
|
||||
std::abort();
|
||||
fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
|
||||
prev_end.DebugString().c_str(),
|
||||
this_begin.DebugString().c_str());
|
||||
abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -721,7 +764,7 @@ class VersionSet::Builder {
|
||||
std::vector<FileMetaData*>* files = &v->files_[level];
|
||||
if (level > 0 && !files->empty()) {
|
||||
// Must not overlap
|
||||
assert(vset_->icmp_.Compare((*files)[files->size() - 1]->largest,
|
||||
assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
|
||||
f->smallest) < 0);
|
||||
}
|
||||
f->refs++;
|
||||
@ -730,7 +773,8 @@ class VersionSet::Builder {
|
||||
}
|
||||
};
|
||||
|
||||
VersionSet::VersionSet(const std::string& dbname, const Options* options,
|
||||
VersionSet::VersionSet(const std::string& dbname,
|
||||
const Options* options,
|
||||
TableCache* table_cache,
|
||||
const InternalKeyComparator* cmp)
|
||||
: env_(options->env),
|
||||
@ -806,6 +850,7 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
|
||||
// first call to LogAndApply (when opening the database).
|
||||
assert(descriptor_file_ == nullptr);
|
||||
new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
|
||||
edit->SetNextFile(next_file_number_);
|
||||
s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
|
||||
if (s.ok()) {
|
||||
descriptor_log_ = new log::Writer(descriptor_file_);
|
||||
@ -851,17 +896,17 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
|
||||
delete descriptor_file_;
|
||||
descriptor_log_ = nullptr;
|
||||
descriptor_file_ = nullptr;
|
||||
env_->RemoveFile(new_manifest_file);
|
||||
env_->DeleteFile(new_manifest_file);
|
||||
}
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
Status VersionSet::Recover(bool* save_manifest) {
|
||||
Status VersionSet::Recover(bool *save_manifest) {
|
||||
struct LogReporter : public log::Reader::Reporter {
|
||||
Status* status;
|
||||
void Corruption(size_t bytes, const Status& s) override {
|
||||
virtual void Corruption(size_t bytes, const Status& s) {
|
||||
if (this->status->ok()) *this->status = s;
|
||||
}
|
||||
};
|
||||
@ -872,7 +917,7 @@ Status VersionSet::Recover(bool* save_manifest) {
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
if (current.empty() || current[current.size() - 1] != '\n') {
|
||||
if (current.empty() || current[current.size()-1] != '\n') {
|
||||
return Status::Corruption("CURRENT file does not end with newline");
|
||||
}
|
||||
current.resize(current.size() - 1);
|
||||
@ -882,8 +927,8 @@ Status VersionSet::Recover(bool* save_manifest) {
|
||||
s = env_->NewSequentialFile(dscname, &file);
|
||||
if (!s.ok()) {
|
||||
if (s.IsNotFound()) {
|
||||
return Status::Corruption("CURRENT points to a non-existent file",
|
||||
s.ToString());
|
||||
return Status::Corruption(
|
||||
"CURRENT points to a non-existent file", s.ToString());
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@ -897,17 +942,14 @@ Status VersionSet::Recover(bool* save_manifest) {
|
||||
uint64_t log_number = 0;
|
||||
uint64_t prev_log_number = 0;
|
||||
Builder builder(this, current_);
|
||||
int read_records = 0;
|
||||
|
||||
{
|
||||
LogReporter reporter;
|
||||
reporter.status = &s;
|
||||
log::Reader reader(file, &reporter, true /*checksum*/,
|
||||
0 /*initial_offset*/);
|
||||
log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
while (reader.ReadRecord(&record, &scratch) && s.ok()) {
|
||||
++read_records;
|
||||
VersionEdit edit;
|
||||
s = edit.DecodeFrom(record);
|
||||
if (s.ok()) {
|
||||
@ -982,10 +1024,6 @@ Status VersionSet::Recover(bool* save_manifest) {
|
||||
} else {
|
||||
*save_manifest = true;
|
||||
}
|
||||
} else {
|
||||
std::string error = s.ToString();
|
||||
Log(options_->info_log, "Error recovering version set with %d records: %s",
|
||||
read_records, error.c_str());
|
||||
}
|
||||
|
||||
return s;
|
||||
@ -1033,7 +1071,7 @@ void VersionSet::Finalize(Version* v) {
|
||||
int best_level = -1;
|
||||
double best_score = -1;
|
||||
|
||||
for (int level = 0; level < config::kNumLevels - 1; level++) {
|
||||
for (int level = 0; level < config::kNumLevels-1; level++) {
|
||||
double score;
|
||||
if (level == 0) {
|
||||
// We treat level-0 specially by bounding the number of files
|
||||
@ -1048,7 +1086,7 @@ void VersionSet::Finalize(Version* v) {
|
||||
// setting, or very high compression ratios, or lots of
|
||||
// overwrites/deletions).
|
||||
score = v->files_[level].size() /
|
||||
static_cast<double>(config::kL0_CompactionTrigger);
|
||||
static_cast<double>(config::kL0_CompactionTrigger);
|
||||
} else {
|
||||
// Compute the ratio of current size to size limit.
|
||||
const uint64_t level_bytes = TotalFileSize(v->files_[level]);
|
||||
@ -1104,13 +1142,16 @@ int VersionSet::NumLevelFiles(int level) const {
|
||||
|
||||
const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
|
||||
// Update code if kNumLevels changes
|
||||
static_assert(config::kNumLevels == 7, "");
|
||||
std::snprintf(
|
||||
scratch->buffer, sizeof(scratch->buffer), "files[ %d %d %d %d %d %d %d ]",
|
||||
int(current_->files_[0].size()), int(current_->files_[1].size()),
|
||||
int(current_->files_[2].size()), int(current_->files_[3].size()),
|
||||
int(current_->files_[4].size()), int(current_->files_[5].size()),
|
||||
int(current_->files_[6].size()));
|
||||
assert(config::kNumLevels == 7);
|
||||
snprintf(scratch->buffer, sizeof(scratch->buffer),
|
||||
"files[ %d %d %d %d %d %d %d ]",
|
||||
int(current_->files_[0].size()),
|
||||
int(current_->files_[1].size()),
|
||||
int(current_->files_[2].size()),
|
||||
int(current_->files_[3].size()),
|
||||
int(current_->files_[4].size()),
|
||||
int(current_->files_[5].size()),
|
||||
int(current_->files_[6].size()));
|
||||
return scratch->buffer;
|
||||
}
|
||||
|
||||
@ -1147,7 +1188,8 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
|
||||
}
|
||||
|
||||
void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
|
||||
for (Version* v = dummy_versions_.next_; v != &dummy_versions_;
|
||||
for (Version* v = dummy_versions_.next_;
|
||||
v != &dummy_versions_;
|
||||
v = v->next_) {
|
||||
for (int level = 0; level < config::kNumLevels; level++) {
|
||||
const std::vector<FileMetaData*>& files = v->files_[level];
|
||||
@ -1170,7 +1212,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() {
|
||||
for (int level = 1; level < config::kNumLevels - 1; level++) {
|
||||
for (size_t i = 0; i < current_->files_[level].size(); i++) {
|
||||
const FileMetaData* f = current_->files_[level][i];
|
||||
current_->GetOverlappingInputs(level + 1, &f->smallest, &f->largest,
|
||||
current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest,
|
||||
&overlaps);
|
||||
const int64_t sum = TotalFileSize(overlaps);
|
||||
if (sum > result) {
|
||||
@ -1185,7 +1227,8 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() {
|
||||
// *smallest, *largest.
|
||||
// REQUIRES: inputs is not empty
|
||||
void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
|
||||
InternalKey* smallest, InternalKey* largest) {
|
||||
InternalKey* smallest,
|
||||
InternalKey* largest) {
|
||||
assert(!inputs.empty());
|
||||
smallest->Clear();
|
||||
largest->Clear();
|
||||
@ -1210,7 +1253,8 @@ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
|
||||
// REQUIRES: inputs is not empty
|
||||
void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
|
||||
const std::vector<FileMetaData*>& inputs2,
|
||||
InternalKey* smallest, InternalKey* largest) {
|
||||
InternalKey* smallest,
|
||||
InternalKey* largest) {
|
||||
std::vector<FileMetaData*> all = inputs1;
|
||||
all.insert(all.end(), inputs2.begin(), inputs2.end());
|
||||
GetRange(all, smallest, largest);
|
||||
@ -1232,8 +1276,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
|
||||
if (c->level() + which == 0) {
|
||||
const std::vector<FileMetaData*>& files = c->inputs_[which];
|
||||
for (size_t i = 0; i < files.size(); i++) {
|
||||
list[num++] = table_cache_->NewIterator(options, files[i]->number,
|
||||
files[i]->file_size);
|
||||
list[num++] = table_cache_->NewIterator(
|
||||
options, files[i]->number, files[i]->file_size);
|
||||
}
|
||||
} else {
|
||||
// Create concatenating iterator for the files from this level
|
||||
@ -1260,7 +1304,7 @@ Compaction* VersionSet::PickCompaction() {
|
||||
if (size_compaction) {
|
||||
level = current_->compaction_level_;
|
||||
assert(level >= 0);
|
||||
assert(level + 1 < config::kNumLevels);
|
||||
assert(level+1 < config::kNumLevels);
|
||||
c = new Compaction(options_, level);
|
||||
|
||||
// Pick the first file that comes after compact_pointer_[level]
|
||||
@ -1303,7 +1347,7 @@ Compaction* VersionSet::PickCompaction() {
|
||||
return c;
|
||||
}
|
||||
|
||||
// Finds the largest key in a vector of files. Returns true if files is not
|
||||
// Finds the largest key in a vector of files. Returns true if files it not
|
||||
// empty.
|
||||
bool FindLargestKey(const InternalKeyComparator& icmp,
|
||||
const std::vector<FileMetaData*>& files,
|
||||
@ -1389,9 +1433,7 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
|
||||
AddBoundaryInputs(icmp_, current_->files_[level], &c->inputs_[0]);
|
||||
GetRange(c->inputs_[0], &smallest, &largest);
|
||||
|
||||
current_->GetOverlappingInputs(level + 1, &smallest, &largest,
|
||||
&c->inputs_[1]);
|
||||
AddBoundaryInputs(icmp_, current_->files_[level + 1], &c->inputs_[1]);
|
||||
current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]);
|
||||
|
||||
// Get entire range covered by compaction
|
||||
InternalKey all_start, all_limit;
|
||||
@ -1412,15 +1454,18 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
|
||||
InternalKey new_start, new_limit;
|
||||
GetRange(expanded0, &new_start, &new_limit);
|
||||
std::vector<FileMetaData*> expanded1;
|
||||
current_->GetOverlappingInputs(level + 1, &new_start, &new_limit,
|
||||
current_->GetOverlappingInputs(level+1, &new_start, &new_limit,
|
||||
&expanded1);
|
||||
AddBoundaryInputs(icmp_, current_->files_[level + 1], &expanded1);
|
||||
if (expanded1.size() == c->inputs_[1].size()) {
|
||||
Log(options_->info_log,
|
||||
"Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
|
||||
level, int(c->inputs_[0].size()), int(c->inputs_[1].size()),
|
||||
long(inputs0_size), long(inputs1_size), int(expanded0.size()),
|
||||
int(expanded1.size()), long(expanded0_size), long(inputs1_size));
|
||||
level,
|
||||
int(c->inputs_[0].size()),
|
||||
int(c->inputs_[1].size()),
|
||||
long(inputs0_size), long(inputs1_size),
|
||||
int(expanded0.size()),
|
||||
int(expanded1.size()),
|
||||
long(expanded0_size), long(inputs1_size));
|
||||
smallest = new_start;
|
||||
largest = new_limit;
|
||||
c->inputs_[0] = expanded0;
|
||||
@ -1445,8 +1490,10 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
|
||||
c->edit_.SetCompactPointer(level, largest);
|
||||
}
|
||||
|
||||
Compaction* VersionSet::CompactRange(int level, const InternalKey* begin,
|
||||
const InternalKey* end) {
|
||||
Compaction* VersionSet::CompactRange(
|
||||
int level,
|
||||
const InternalKey* begin,
|
||||
const InternalKey* end) {
|
||||
std::vector<FileMetaData*> inputs;
|
||||
current_->GetOverlappingInputs(level, begin, end, &inputs);
|
||||
if (inputs.empty()) {
|
||||
@ -1509,7 +1556,7 @@ bool Compaction::IsTrivialMove() const {
|
||||
void Compaction::AddInputDeletions(VersionEdit* edit) {
|
||||
for (int which = 0; which < 2; which++) {
|
||||
for (size_t i = 0; i < inputs_[which].size(); i++) {
|
||||
edit->RemoveFile(level_ + which, inputs_[which][i]->number);
|
||||
edit->DeleteFile(level_ + which, inputs_[which][i]->number);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1519,7 +1566,7 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
|
||||
const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
|
||||
for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
|
||||
const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
|
||||
while (level_ptrs_[lvl] < files.size()) {
|
||||
for (; level_ptrs_[lvl] < files.size(); ) {
|
||||
FileMetaData* f = files[level_ptrs_[lvl]];
|
||||
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
|
||||
// We've advanced far enough
|
||||
@ -1540,9 +1587,8 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
|
||||
// Scan to find earliest grandparent file that contains key.
|
||||
const InternalKeyComparator* icmp = &vset->icmp_;
|
||||
while (grandparent_index_ < grandparents_.size() &&
|
||||
icmp->Compare(internal_key,
|
||||
grandparents_[grandparent_index_]->largest.Encode()) >
|
||||
0) {
|
||||
icmp->Compare(internal_key,
|
||||
grandparents_[grandparent_index_]->largest.Encode()) > 0) {
|
||||
if (seen_key_) {
|
||||
overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
|
||||
}
|
||||
|
@ -18,7 +18,6 @@
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "port/port.h"
|
||||
@ -26,9 +25,7 @@
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace log {
|
||||
class Writer;
|
||||
}
|
||||
namespace log { class Writer; }
|
||||
|
||||
class Compaction;
|
||||
class Iterator;
|
||||
@ -43,7 +40,8 @@ class WritableFile;
|
||||
// Return files.size() if there is no such file.
|
||||
// REQUIRES: "files" contains a sorted list of non-overlapping files.
|
||||
int FindFile(const InternalKeyComparator& icmp,
|
||||
const std::vector<FileMetaData*>& files, const Slice& key);
|
||||
const std::vector<FileMetaData*>& files,
|
||||
const Slice& key);
|
||||
|
||||
// Returns true iff some file in "files" overlaps the user key range
|
||||
// [*smallest,*largest].
|
||||
@ -59,11 +57,6 @@ bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
|
||||
|
||||
class Version {
|
||||
public:
|
||||
struct GetStats {
|
||||
FileMetaData* seek_file;
|
||||
int seek_file_level;
|
||||
};
|
||||
|
||||
// Append to *iters a sequence of iterators that will
|
||||
// yield the contents of this Version when merged together.
|
||||
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
||||
@ -72,6 +65,10 @@ class Version {
|
||||
// Lookup the value for key. If found, store it in *val and
|
||||
// return OK. Else return a non-OK status. Fills *stats.
|
||||
// REQUIRES: lock is not held
|
||||
struct GetStats {
|
||||
FileMetaData* seek_file;
|
||||
int seek_file_level;
|
||||
};
|
||||
Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
|
||||
GetStats* stats);
|
||||
|
||||
@ -93,15 +90,16 @@ class Version {
|
||||
|
||||
void GetOverlappingInputs(
|
||||
int level,
|
||||
const InternalKey* begin, // nullptr means before all keys
|
||||
const InternalKey* end, // nullptr means after all keys
|
||||
const InternalKey* begin, // nullptr means before all keys
|
||||
const InternalKey* end, // nullptr means after all keys
|
||||
std::vector<FileMetaData*>* inputs);
|
||||
|
||||
// Returns true iff some file in the specified level overlaps
|
||||
// some part of [*smallest_user_key,*largest_user_key].
|
||||
// smallest_user_key==nullptr represents a key smaller than all the DB's keys.
|
||||
// largest_user_key==nullptr represents a key largest than all the DB's keys.
|
||||
bool OverlapInLevel(int level, const Slice* smallest_user_key,
|
||||
bool OverlapInLevel(int level,
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key);
|
||||
|
||||
// Return the level at which we should place a new memtable compaction
|
||||
@ -119,22 +117,6 @@ class Version {
|
||||
friend class VersionSet;
|
||||
|
||||
class LevelFileNumIterator;
|
||||
|
||||
explicit Version(VersionSet* vset)
|
||||
: vset_(vset),
|
||||
next_(this),
|
||||
prev_(this),
|
||||
refs_(0),
|
||||
file_to_compact_(nullptr),
|
||||
file_to_compact_level_(-1),
|
||||
compaction_score_(-1),
|
||||
compaction_level_(-1) {}
|
||||
|
||||
Version(const Version&) = delete;
|
||||
Version& operator=(const Version&) = delete;
|
||||
|
||||
~Version();
|
||||
|
||||
Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
|
||||
|
||||
// Call func(arg, level, f) for every file that overlaps user_key in
|
||||
@ -142,13 +124,14 @@ class Version {
|
||||
// false, makes no more calls.
|
||||
//
|
||||
// REQUIRES: user portion of internal_key == user_key.
|
||||
void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
|
||||
void ForEachOverlapping(Slice user_key, Slice internal_key,
|
||||
void* arg,
|
||||
bool (*func)(void*, int, FileMetaData*));
|
||||
|
||||
VersionSet* vset_; // VersionSet to which this Version belongs
|
||||
Version* next_; // Next version in linked list
|
||||
Version* prev_; // Previous version in linked list
|
||||
int refs_; // Number of live refs to this version
|
||||
VersionSet* vset_; // VersionSet to which this Version belongs
|
||||
Version* next_; // Next version in linked list
|
||||
Version* prev_; // Previous version in linked list
|
||||
int refs_; // Number of live refs to this version
|
||||
|
||||
// List of files per level
|
||||
std::vector<FileMetaData*> files_[config::kNumLevels];
|
||||
@ -162,15 +145,28 @@ class Version {
|
||||
// are initialized by Finalize().
|
||||
double compaction_score_;
|
||||
int compaction_level_;
|
||||
|
||||
explicit Version(VersionSet* vset)
|
||||
: vset_(vset), next_(this), prev_(this), refs_(0),
|
||||
file_to_compact_(nullptr),
|
||||
file_to_compact_level_(-1),
|
||||
compaction_score_(-1),
|
||||
compaction_level_(-1) {
|
||||
}
|
||||
|
||||
~Version();
|
||||
|
||||
// No copying allowed
|
||||
Version(const Version&);
|
||||
void operator=(const Version&);
|
||||
};
|
||||
|
||||
class VersionSet {
|
||||
public:
|
||||
VersionSet(const std::string& dbname, const Options* options,
|
||||
TableCache* table_cache, const InternalKeyComparator*);
|
||||
VersionSet(const VersionSet&) = delete;
|
||||
VersionSet& operator=(const VersionSet&) = delete;
|
||||
|
||||
VersionSet(const std::string& dbname,
|
||||
const Options* options,
|
||||
TableCache* table_cache,
|
||||
const InternalKeyComparator*);
|
||||
~VersionSet();
|
||||
|
||||
// Apply *edit to the current version to form a new descriptor that
|
||||
@ -182,7 +178,7 @@ class VersionSet {
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu);
|
||||
|
||||
// Recover the last saved descriptor from persistent storage.
|
||||
Status Recover(bool* save_manifest);
|
||||
Status Recover(bool *save_manifest);
|
||||
|
||||
// Return the current version.
|
||||
Version* current() const { return current_; }
|
||||
@ -237,8 +233,10 @@ class VersionSet {
|
||||
// the specified level. Returns nullptr if there is nothing in that
|
||||
// level that overlaps the specified range. Caller should delete
|
||||
// the result.
|
||||
Compaction* CompactRange(int level, const InternalKey* begin,
|
||||
const InternalKey* end);
|
||||
Compaction* CompactRange(
|
||||
int level,
|
||||
const InternalKey* begin,
|
||||
const InternalKey* end);
|
||||
|
||||
// Return the maximum overlapping data (in bytes) at next level for any
|
||||
// file at a level >= 1.
|
||||
@ -279,12 +277,14 @@ class VersionSet {
|
||||
|
||||
void Finalize(Version* v);
|
||||
|
||||
void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest,
|
||||
void GetRange(const std::vector<FileMetaData*>& inputs,
|
||||
InternalKey* smallest,
|
||||
InternalKey* largest);
|
||||
|
||||
void GetRange2(const std::vector<FileMetaData*>& inputs1,
|
||||
const std::vector<FileMetaData*>& inputs2,
|
||||
InternalKey* smallest, InternalKey* largest);
|
||||
InternalKey* smallest,
|
||||
InternalKey* largest);
|
||||
|
||||
void SetupOtherInputs(Compaction* c);
|
||||
|
||||
@ -313,6 +313,10 @@ class VersionSet {
|
||||
// Per-level key at which the next compaction at that level should start.
|
||||
// Either an empty string, or a valid InternalKey.
|
||||
std::string compact_pointer_[config::kNumLevels];
|
||||
|
||||
// No copying allowed
|
||||
VersionSet(const VersionSet&);
|
||||
void operator=(const VersionSet&);
|
||||
};
|
||||
|
||||
// A Compaction encapsulates information about a compaction.
|
||||
@ -369,7 +373,7 @@ class Compaction {
|
||||
VersionEdit edit_;
|
||||
|
||||
// Each compaction reads inputs from "level_" and "level_+1"
|
||||
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
|
||||
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
|
||||
|
||||
// State used to check for number of overlapping grandparent files
|
||||
// (parent == level_ + 1, grandparent == level_ + 2)
|
||||
|
@ -3,16 +3,18 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/version_set.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class FindFileTest : public testing::Test {
|
||||
class FindFileTest {
|
||||
public:
|
||||
FindFileTest() : disjoint_sorted_files_(true) {}
|
||||
std::vector<FileMetaData*> files_;
|
||||
bool disjoint_sorted_files_;
|
||||
|
||||
FindFileTest() : disjoint_sorted_files_(true) { }
|
||||
|
||||
~FindFileTest() {
|
||||
for (int i = 0; i < files_.size(); i++) {
|
||||
@ -44,22 +46,17 @@ class FindFileTest : public testing::Test {
|
||||
(smallest != nullptr ? &s : nullptr),
|
||||
(largest != nullptr ? &l : nullptr));
|
||||
}
|
||||
|
||||
bool disjoint_sorted_files_;
|
||||
|
||||
private:
|
||||
std::vector<FileMetaData*> files_;
|
||||
};
|
||||
|
||||
TEST_F(FindFileTest, Empty) {
|
||||
TEST(FindFileTest, Empty) {
|
||||
ASSERT_EQ(0, Find("foo"));
|
||||
ASSERT_TRUE(!Overlaps("a", "z"));
|
||||
ASSERT_TRUE(!Overlaps(nullptr, "z"));
|
||||
ASSERT_TRUE(!Overlaps("a", nullptr));
|
||||
ASSERT_TRUE(!Overlaps(nullptr, nullptr));
|
||||
ASSERT_TRUE(! Overlaps("a", "z"));
|
||||
ASSERT_TRUE(! Overlaps(nullptr, "z"));
|
||||
ASSERT_TRUE(! Overlaps("a", nullptr));
|
||||
ASSERT_TRUE(! Overlaps(nullptr, nullptr));
|
||||
}
|
||||
|
||||
TEST_F(FindFileTest, Single) {
|
||||
TEST(FindFileTest, Single) {
|
||||
Add("p", "q");
|
||||
ASSERT_EQ(0, Find("a"));
|
||||
ASSERT_EQ(0, Find("p"));
|
||||
@ -68,8 +65,8 @@ TEST_F(FindFileTest, Single) {
|
||||
ASSERT_EQ(1, Find("q1"));
|
||||
ASSERT_EQ(1, Find("z"));
|
||||
|
||||
ASSERT_TRUE(!Overlaps("a", "b"));
|
||||
ASSERT_TRUE(!Overlaps("z1", "z2"));
|
||||
ASSERT_TRUE(! Overlaps("a", "b"));
|
||||
ASSERT_TRUE(! Overlaps("z1", "z2"));
|
||||
ASSERT_TRUE(Overlaps("a", "p"));
|
||||
ASSERT_TRUE(Overlaps("a", "q"));
|
||||
ASSERT_TRUE(Overlaps("a", "z"));
|
||||
@ -81,15 +78,16 @@ TEST_F(FindFileTest, Single) {
|
||||
ASSERT_TRUE(Overlaps("q", "q"));
|
||||
ASSERT_TRUE(Overlaps("q", "q1"));
|
||||
|
||||
ASSERT_TRUE(!Overlaps(nullptr, "j"));
|
||||
ASSERT_TRUE(!Overlaps("r", nullptr));
|
||||
ASSERT_TRUE(! Overlaps(nullptr, "j"));
|
||||
ASSERT_TRUE(! Overlaps("r", nullptr));
|
||||
ASSERT_TRUE(Overlaps(nullptr, "p"));
|
||||
ASSERT_TRUE(Overlaps(nullptr, "p1"));
|
||||
ASSERT_TRUE(Overlaps("q", nullptr));
|
||||
ASSERT_TRUE(Overlaps(nullptr, nullptr));
|
||||
}
|
||||
|
||||
TEST_F(FindFileTest, Multiple) {
|
||||
|
||||
TEST(FindFileTest, Multiple) {
|
||||
Add("150", "200");
|
||||
Add("200", "250");
|
||||
Add("300", "350");
|
||||
@ -112,10 +110,10 @@ TEST_F(FindFileTest, Multiple) {
|
||||
ASSERT_EQ(3, Find("450"));
|
||||
ASSERT_EQ(4, Find("451"));
|
||||
|
||||
ASSERT_TRUE(!Overlaps("100", "149"));
|
||||
ASSERT_TRUE(!Overlaps("251", "299"));
|
||||
ASSERT_TRUE(!Overlaps("451", "500"));
|
||||
ASSERT_TRUE(!Overlaps("351", "399"));
|
||||
ASSERT_TRUE(! Overlaps("100", "149"));
|
||||
ASSERT_TRUE(! Overlaps("251", "299"));
|
||||
ASSERT_TRUE(! Overlaps("451", "500"));
|
||||
ASSERT_TRUE(! Overlaps("351", "399"));
|
||||
|
||||
ASSERT_TRUE(Overlaps("100", "150"));
|
||||
ASSERT_TRUE(Overlaps("100", "200"));
|
||||
@ -127,13 +125,13 @@ TEST_F(FindFileTest, Multiple) {
|
||||
ASSERT_TRUE(Overlaps("450", "500"));
|
||||
}
|
||||
|
||||
TEST_F(FindFileTest, MultipleNullBoundaries) {
|
||||
TEST(FindFileTest, MultipleNullBoundaries) {
|
||||
Add("150", "200");
|
||||
Add("200", "250");
|
||||
Add("300", "350");
|
||||
Add("400", "450");
|
||||
ASSERT_TRUE(!Overlaps(nullptr, "149"));
|
||||
ASSERT_TRUE(!Overlaps("451", nullptr));
|
||||
ASSERT_TRUE(! Overlaps(nullptr, "149"));
|
||||
ASSERT_TRUE(! Overlaps("451", nullptr));
|
||||
ASSERT_TRUE(Overlaps(nullptr, nullptr));
|
||||
ASSERT_TRUE(Overlaps(nullptr, "150"));
|
||||
ASSERT_TRUE(Overlaps(nullptr, "199"));
|
||||
@ -147,21 +145,21 @@ TEST_F(FindFileTest, MultipleNullBoundaries) {
|
||||
ASSERT_TRUE(Overlaps("450", nullptr));
|
||||
}
|
||||
|
||||
TEST_F(FindFileTest, OverlapSequenceChecks) {
|
||||
TEST(FindFileTest, OverlapSequenceChecks) {
|
||||
Add("200", "200", 5000, 3000);
|
||||
ASSERT_TRUE(!Overlaps("199", "199"));
|
||||
ASSERT_TRUE(!Overlaps("201", "300"));
|
||||
ASSERT_TRUE(! Overlaps("199", "199"));
|
||||
ASSERT_TRUE(! Overlaps("201", "300"));
|
||||
ASSERT_TRUE(Overlaps("200", "200"));
|
||||
ASSERT_TRUE(Overlaps("190", "200"));
|
||||
ASSERT_TRUE(Overlaps("200", "210"));
|
||||
}
|
||||
|
||||
TEST_F(FindFileTest, OverlappingFiles) {
|
||||
TEST(FindFileTest, OverlappingFiles) {
|
||||
Add("150", "600");
|
||||
Add("400", "500");
|
||||
disjoint_sorted_files_ = false;
|
||||
ASSERT_TRUE(!Overlaps("100", "149"));
|
||||
ASSERT_TRUE(!Overlaps("601", "700"));
|
||||
ASSERT_TRUE(! Overlaps("100", "149"));
|
||||
ASSERT_TRUE(! Overlaps("601", "700"));
|
||||
ASSERT_TRUE(Overlaps("100", "150"));
|
||||
ASSERT_TRUE(Overlaps("100", "200"));
|
||||
ASSERT_TRUE(Overlaps("100", "300"));
|
||||
@ -178,21 +176,21 @@ void AddBoundaryInputs(const InternalKeyComparator& icmp,
|
||||
const std::vector<FileMetaData*>& level_files,
|
||||
std::vector<FileMetaData*>* compaction_files);
|
||||
|
||||
class AddBoundaryInputsTest : public testing::Test {
|
||||
class AddBoundaryInputsTest {
|
||||
public:
|
||||
std::vector<FileMetaData*> level_files_;
|
||||
std::vector<FileMetaData*> compaction_files_;
|
||||
std::vector<FileMetaData*> all_files_;
|
||||
InternalKeyComparator icmp_;
|
||||
|
||||
AddBoundaryInputsTest() : icmp_(BytewiseComparator()) {}
|
||||
AddBoundaryInputsTest() : icmp_(BytewiseComparator()){};
|
||||
|
||||
~AddBoundaryInputsTest() {
|
||||
for (size_t i = 0; i < all_files_.size(); ++i) {
|
||||
delete all_files_[i];
|
||||
}
|
||||
all_files_.clear();
|
||||
}
|
||||
};
|
||||
|
||||
FileMetaData* CreateFileMetaData(uint64_t number, InternalKey smallest,
|
||||
InternalKey largest) {
|
||||
@ -205,13 +203,13 @@ class AddBoundaryInputsTest : public testing::Test {
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestEmptyFileSets) {
|
||||
TEST(AddBoundaryInputsTest, TestEmptyFileSets) {
|
||||
AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
|
||||
ASSERT_TRUE(compaction_files_.empty());
|
||||
ASSERT_TRUE(level_files_.empty());
|
||||
}
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestEmptyLevelFiles) {
|
||||
TEST(AddBoundaryInputsTest, TestEmptyLevelFiles) {
|
||||
FileMetaData* f1 =
|
||||
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
|
||||
InternalKey(InternalKey("100", 1, kTypeValue)));
|
||||
@ -223,7 +221,7 @@ TEST_F(AddBoundaryInputsTest, TestEmptyLevelFiles) {
|
||||
ASSERT_TRUE(level_files_.empty());
|
||||
}
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestEmptyCompactionFiles) {
|
||||
TEST(AddBoundaryInputsTest, TestEmptyCompactionFiles) {
|
||||
FileMetaData* f1 =
|
||||
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
|
||||
InternalKey(InternalKey("100", 1, kTypeValue)));
|
||||
@ -235,7 +233,7 @@ TEST_F(AddBoundaryInputsTest, TestEmptyCompactionFiles) {
|
||||
ASSERT_EQ(f1, level_files_[0]);
|
||||
}
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestNoBoundaryFiles) {
|
||||
TEST(AddBoundaryInputsTest, TestNoBoundaryFiles) {
|
||||
FileMetaData* f1 =
|
||||
CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
|
||||
InternalKey(InternalKey("100", 1, kTypeValue)));
|
||||
@ -256,7 +254,7 @@ TEST_F(AddBoundaryInputsTest, TestNoBoundaryFiles) {
|
||||
ASSERT_EQ(2, compaction_files_.size());
|
||||
}
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestOneBoundaryFiles) {
|
||||
TEST(AddBoundaryInputsTest, TestOneBoundaryFiles) {
|
||||
FileMetaData* f1 =
|
||||
CreateFileMetaData(1, InternalKey("100", 3, kTypeValue),
|
||||
InternalKey(InternalKey("100", 2, kTypeValue)));
|
||||
@ -278,7 +276,7 @@ TEST_F(AddBoundaryInputsTest, TestOneBoundaryFiles) {
|
||||
ASSERT_EQ(f2, compaction_files_[1]);
|
||||
}
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestTwoBoundaryFiles) {
|
||||
TEST(AddBoundaryInputsTest, TestTwoBoundaryFiles) {
|
||||
FileMetaData* f1 =
|
||||
CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
|
||||
InternalKey(InternalKey("100", 5, kTypeValue)));
|
||||
@ -301,7 +299,7 @@ TEST_F(AddBoundaryInputsTest, TestTwoBoundaryFiles) {
|
||||
ASSERT_EQ(f2, compaction_files_[2]);
|
||||
}
|
||||
|
||||
TEST_F(AddBoundaryInputsTest, TestDisjoinFilePointers) {
|
||||
TEST(AddBoundaryInputsTest, TestDisjoinFilePointers) {
|
||||
FileMetaData* f1 =
|
||||
CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
|
||||
InternalKey(InternalKey("100", 5, kTypeValue)));
|
||||
@ -329,3 +327,5 @@ TEST_F(AddBoundaryInputsTest, TestDisjoinFilePointers) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
|
||||
|
@ -15,10 +15,10 @@
|
||||
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/memtable.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -26,18 +26,22 @@ namespace leveldb {
|
||||
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
||||
static const size_t kHeader = 12;
|
||||
|
||||
WriteBatch::WriteBatch() { Clear(); }
|
||||
WriteBatch::WriteBatch() {
|
||||
Clear();
|
||||
}
|
||||
|
||||
WriteBatch::~WriteBatch() = default;
|
||||
WriteBatch::~WriteBatch() { }
|
||||
|
||||
WriteBatch::Handler::~Handler() = default;
|
||||
WriteBatch::Handler::~Handler() { }
|
||||
|
||||
void WriteBatch::Clear() {
|
||||
rep_.clear();
|
||||
rep_.resize(kHeader);
|
||||
}
|
||||
|
||||
size_t WriteBatch::ApproximateSize() const { return rep_.size(); }
|
||||
size_t WriteBatch::ApproximateSize() const {
|
||||
return rep_.size();
|
||||
}
|
||||
|
||||
Status WriteBatch::Iterate(Handler* handler) const {
|
||||
Slice input(rep_);
|
||||
@ -108,7 +112,7 @@ void WriteBatch::Delete(const Slice& key) {
|
||||
PutLengthPrefixedSlice(&rep_, key);
|
||||
}
|
||||
|
||||
void WriteBatch::Append(const WriteBatch& source) {
|
||||
void WriteBatch::Append(const WriteBatch &source) {
|
||||
WriteBatchInternal::Append(this, &source);
|
||||
}
|
||||
|
||||
@ -118,18 +122,19 @@ class MemTableInserter : public WriteBatch::Handler {
|
||||
SequenceNumber sequence_;
|
||||
MemTable* mem_;
|
||||
|
||||
void Put(const Slice& key, const Slice& value) override {
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
mem_->Add(sequence_, kTypeValue, key, value);
|
||||
sequence_++;
|
||||
}
|
||||
void Delete(const Slice& key) override {
|
||||
virtual void Delete(const Slice& key) {
|
||||
mem_->Add(sequence_, kTypeDeletion, key, Slice());
|
||||
sequence_++;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) {
|
||||
Status WriteBatchInternal::InsertInto(const WriteBatch* b,
|
||||
MemTable* memtable) {
|
||||
MemTableInserter inserter;
|
||||
inserter.sequence_ = WriteBatchInternal::Sequence(b);
|
||||
inserter.mem_ = memtable;
|
||||
|
@ -29,9 +29,13 @@ class WriteBatchInternal {
|
||||
// this batch.
|
||||
static void SetSequence(WriteBatch* batch, SequenceNumber seq);
|
||||
|
||||
static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); }
|
||||
static Slice Contents(const WriteBatch* batch) {
|
||||
return Slice(batch->rep_);
|
||||
}
|
||||
|
||||
static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); }
|
||||
static size_t ByteSize(const WriteBatch* batch) {
|
||||
return batch->rep_.size();
|
||||
}
|
||||
|
||||
static void SetContents(WriteBatch* batch, const Slice& contents);
|
||||
|
||||
@ -42,4 +46,5 @@ class WriteBatchInternal {
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
|
@ -2,12 +2,13 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/db.h"
|
||||
|
||||
#include "db/memtable.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@ -21,7 +22,7 @@ static std::string PrintContents(WriteBatch* b) {
|
||||
Iterator* iter = mem->NewIterator();
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ParsedInternalKey ikey;
|
||||
EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey));
|
||||
ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey));
|
||||
switch (ikey.type) {
|
||||
case kTypeValue:
|
||||
state.append("Put(");
|
||||
@ -51,6 +52,8 @@ static std::string PrintContents(WriteBatch* b) {
|
||||
return state;
|
||||
}
|
||||
|
||||
class WriteBatchTest { };
|
||||
|
||||
TEST(WriteBatchTest, Empty) {
|
||||
WriteBatch batch;
|
||||
ASSERT_EQ("", PrintContents(&batch));
|
||||
@ -65,11 +68,10 @@ TEST(WriteBatchTest, Multiple) {
|
||||
WriteBatchInternal::SetSequence(&batch, 100);
|
||||
ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch));
|
||||
ASSERT_EQ(3, WriteBatchInternal::Count(&batch));
|
||||
ASSERT_EQ(
|
||||
"Put(baz, boo)@102"
|
||||
"Delete(box)@101"
|
||||
"Put(foo, bar)@100",
|
||||
PrintContents(&batch));
|
||||
ASSERT_EQ("Put(baz, boo)@102"
|
||||
"Delete(box)@101"
|
||||
"Put(foo, bar)@100",
|
||||
PrintContents(&batch));
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, Corruption) {
|
||||
@ -79,11 +81,10 @@ TEST(WriteBatchTest, Corruption) {
|
||||
WriteBatchInternal::SetSequence(&batch, 200);
|
||||
Slice contents = WriteBatchInternal::Contents(&batch);
|
||||
WriteBatchInternal::SetContents(&batch,
|
||||
Slice(contents.data(), contents.size() - 1));
|
||||
ASSERT_EQ(
|
||||
"Put(foo, bar)@200"
|
||||
"ParseError()",
|
||||
PrintContents(&batch));
|
||||
Slice(contents.data(),contents.size()-1));
|
||||
ASSERT_EQ("Put(foo, bar)@200"
|
||||
"ParseError()",
|
||||
PrintContents(&batch));
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, Append) {
|
||||
@ -91,25 +92,25 @@ TEST(WriteBatchTest, Append) {
|
||||
WriteBatchInternal::SetSequence(&b1, 200);
|
||||
WriteBatchInternal::SetSequence(&b2, 300);
|
||||
b1.Append(b2);
|
||||
ASSERT_EQ("", PrintContents(&b1));
|
||||
ASSERT_EQ("",
|
||||
PrintContents(&b1));
|
||||
b2.Put("a", "va");
|
||||
b1.Append(b2);
|
||||
ASSERT_EQ("Put(a, va)@200", PrintContents(&b1));
|
||||
ASSERT_EQ("Put(a, va)@200",
|
||||
PrintContents(&b1));
|
||||
b2.Clear();
|
||||
b2.Put("b", "vb");
|
||||
b1.Append(b2);
|
||||
ASSERT_EQ(
|
||||
"Put(a, va)@200"
|
||||
"Put(b, vb)@201",
|
||||
PrintContents(&b1));
|
||||
ASSERT_EQ("Put(a, va)@200"
|
||||
"Put(b, vb)@201",
|
||||
PrintContents(&b1));
|
||||
b2.Delete("foo");
|
||||
b1.Append(b2);
|
||||
ASSERT_EQ(
|
||||
"Put(a, va)@200"
|
||||
"Put(b, vb)@202"
|
||||
"Put(b, vb)@201"
|
||||
"Delete(foo)@203",
|
||||
PrintContents(&b1));
|
||||
ASSERT_EQ("Put(a, va)@200"
|
||||
"Put(b, vb)@202"
|
||||
"Put(b, vb)@201"
|
||||
"Delete(foo)@203",
|
||||
PrintContents(&b1));
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, ApproximateSize) {
|
||||
@ -130,3 +131,7 @@ TEST(WriteBatchTest, ApproximateSize) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -2,11 +2,9 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "util/histogram.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testutil.h"
|
||||
@ -40,7 +38,8 @@ static const char* FLAGS_benchmarks =
|
||||
"fillrand100K,"
|
||||
"fillseq100K,"
|
||||
"readseq,"
|
||||
"readrand100K,";
|
||||
"readrand100K,"
|
||||
;
|
||||
|
||||
// Number of key/values to place in database
|
||||
static int FLAGS_num = 1000000;
|
||||
@ -70,9 +69,6 @@ static int FLAGS_num_pages = 4096;
|
||||
// benchmark will fail.
|
||||
static bool FLAGS_use_existing_db = false;
|
||||
|
||||
// If true, the SQLite table has ROWIDs.
|
||||
static bool FLAGS_use_rowids = false;
|
||||
|
||||
// If true, we allow batch writes to occur
|
||||
static bool FLAGS_transaction = true;
|
||||
|
||||
@ -82,29 +78,33 @@ static bool FLAGS_WAL_enabled = true;
|
||||
// Use the db with the following name.
|
||||
static const char* FLAGS_db = nullptr;
|
||||
|
||||
inline static void ExecErrorCheck(int status, char* err_msg) {
|
||||
inline
|
||||
static void ExecErrorCheck(int status, char *err_msg) {
|
||||
if (status != SQLITE_OK) {
|
||||
std::fprintf(stderr, "SQL error: %s\n", err_msg);
|
||||
fprintf(stderr, "SQL error: %s\n", err_msg);
|
||||
sqlite3_free(err_msg);
|
||||
std::exit(1);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
inline static void StepErrorCheck(int status) {
|
||||
inline
|
||||
static void StepErrorCheck(int status) {
|
||||
if (status != SQLITE_DONE) {
|
||||
std::fprintf(stderr, "SQL step error: status = %d\n", status);
|
||||
std::exit(1);
|
||||
fprintf(stderr, "SQL step error: status = %d\n", status);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
inline static void ErrorCheck(int status) {
|
||||
inline
|
||||
static void ErrorCheck(int status) {
|
||||
if (status != SQLITE_OK) {
|
||||
std::fprintf(stderr, "sqlite3 error: status = %d\n", status);
|
||||
std::exit(1);
|
||||
fprintf(stderr, "sqlite3 error: status = %d\n", status);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
inline static void WalCheckpoint(sqlite3* db_) {
|
||||
inline
|
||||
static void WalCheckpoint(sqlite3* db_) {
|
||||
// Flush all writes to disk
|
||||
if (FLAGS_WAL_enabled) {
|
||||
sqlite3_wal_checkpoint_v2(db_, nullptr, SQLITE_CHECKPOINT_FULL, nullptr,
|
||||
@ -153,7 +153,7 @@ static Slice TrimSpace(Slice s) {
|
||||
start++;
|
||||
}
|
||||
int limit = s.size();
|
||||
while (limit > start && isspace(s[limit - 1])) {
|
||||
while (limit > start && isspace(s[limit-1])) {
|
||||
limit--;
|
||||
}
|
||||
return Slice(s.data() + start, limit - start);
|
||||
@ -177,43 +177,41 @@ class Benchmark {
|
||||
|
||||
// State kept for progress messages
|
||||
int done_;
|
||||
int next_report_; // When to report next
|
||||
int next_report_; // When to report next
|
||||
|
||||
void PrintHeader() {
|
||||
const int kKeySize = 16;
|
||||
PrintEnvironment();
|
||||
std::fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
|
||||
std::fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size);
|
||||
std::fprintf(stdout, "Entries: %d\n", num_);
|
||||
std::fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
|
||||
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
|
||||
1048576.0));
|
||||
fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
|
||||
fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size);
|
||||
fprintf(stdout, "Entries: %d\n", num_);
|
||||
fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
|
||||
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
|
||||
/ 1048576.0));
|
||||
PrintWarnings();
|
||||
std::fprintf(stdout, "------------------------------------------------\n");
|
||||
fprintf(stdout, "------------------------------------------------\n");
|
||||
}
|
||||
|
||||
void PrintWarnings() {
|
||||
#if defined(__GNUC__) && !defined(__OPTIMIZE__)
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
|
||||
fprintf(stdout,
|
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
|
||||
);
|
||||
#endif
|
||||
#ifndef NDEBUG
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
|
||||
fprintf(stdout,
|
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
void PrintEnvironment() {
|
||||
std::fprintf(stderr, "SQLite: version %s\n", SQLITE_VERSION);
|
||||
fprintf(stderr, "SQLite: version %s\n", SQLITE_VERSION);
|
||||
|
||||
#if defined(__linux)
|
||||
time_t now = time(nullptr);
|
||||
std::fprintf(stderr, "Date: %s",
|
||||
ctime(&now)); // ctime() adds newline
|
||||
fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
|
||||
|
||||
FILE* cpuinfo = std::fopen("/proc/cpuinfo", "r");
|
||||
FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
|
||||
if (cpuinfo != nullptr) {
|
||||
char line[1000];
|
||||
int num_cpus = 0;
|
||||
@ -233,9 +231,9 @@ class Benchmark {
|
||||
cache_size = val.ToString();
|
||||
}
|
||||
}
|
||||
std::fclose(cpuinfo);
|
||||
std::fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
|
||||
std::fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
|
||||
fclose(cpuinfo);
|
||||
fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
|
||||
fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -256,30 +254,23 @@ class Benchmark {
|
||||
double micros = (now - last_op_finish_) * 1e6;
|
||||
hist_.Add(micros);
|
||||
if (micros > 20000) {
|
||||
std::fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
|
||||
std::fflush(stderr);
|
||||
fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
|
||||
fflush(stderr);
|
||||
}
|
||||
last_op_finish_ = now;
|
||||
}
|
||||
|
||||
done_++;
|
||||
if (done_ >= next_report_) {
|
||||
if (next_report_ < 1000)
|
||||
next_report_ += 100;
|
||||
else if (next_report_ < 5000)
|
||||
next_report_ += 500;
|
||||
else if (next_report_ < 10000)
|
||||
next_report_ += 1000;
|
||||
else if (next_report_ < 50000)
|
||||
next_report_ += 5000;
|
||||
else if (next_report_ < 100000)
|
||||
next_report_ += 10000;
|
||||
else if (next_report_ < 500000)
|
||||
next_report_ += 50000;
|
||||
else
|
||||
next_report_ += 100000;
|
||||
std::fprintf(stderr, "... finished %d ops%30s\r", done_, "");
|
||||
std::fflush(stderr);
|
||||
if (next_report_ < 1000) next_report_ += 100;
|
||||
else if (next_report_ < 5000) next_report_ += 500;
|
||||
else if (next_report_ < 10000) next_report_ += 1000;
|
||||
else if (next_report_ < 50000) next_report_ += 5000;
|
||||
else if (next_report_ < 100000) next_report_ += 10000;
|
||||
else if (next_report_ < 500000) next_report_ += 50000;
|
||||
else next_report_ += 100000;
|
||||
fprintf(stderr, "... finished %d ops%30s\r", done_, "");
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
|
||||
@ -292,36 +283,43 @@ class Benchmark {
|
||||
|
||||
if (bytes_ > 0) {
|
||||
char rate[100];
|
||||
std::snprintf(rate, sizeof(rate), "%6.1f MB/s",
|
||||
(bytes_ / 1048576.0) / (finish - start_));
|
||||
snprintf(rate, sizeof(rate), "%6.1f MB/s",
|
||||
(bytes_ / 1048576.0) / (finish - start_));
|
||||
if (!message_.empty()) {
|
||||
message_ = std::string(rate) + " " + message_;
|
||||
message_ = std::string(rate) + " " + message_;
|
||||
} else {
|
||||
message_ = rate;
|
||||
}
|
||||
}
|
||||
|
||||
std::fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
|
||||
name.ToString().c_str(), (finish - start_) * 1e6 / done_,
|
||||
(message_.empty() ? "" : " "), message_.c_str());
|
||||
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
|
||||
name.ToString().c_str(),
|
||||
(finish - start_) * 1e6 / done_,
|
||||
(message_.empty() ? "" : " "),
|
||||
message_.c_str());
|
||||
if (FLAGS_histogram) {
|
||||
std::fprintf(stdout, "Microseconds per op:\n%s\n",
|
||||
hist_.ToString().c_str());
|
||||
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
|
||||
}
|
||||
std::fflush(stdout);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
public:
|
||||
enum Order { SEQUENTIAL, RANDOM };
|
||||
enum DBState { FRESH, EXISTING };
|
||||
enum Order {
|
||||
SEQUENTIAL,
|
||||
RANDOM
|
||||
};
|
||||
enum DBState {
|
||||
FRESH,
|
||||
EXISTING
|
||||
};
|
||||
|
||||
Benchmark()
|
||||
: db_(nullptr),
|
||||
db_num_(0),
|
||||
num_(FLAGS_num),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
bytes_(0),
|
||||
rand_(301) {
|
||||
: db_(nullptr),
|
||||
db_num_(0),
|
||||
num_(FLAGS_num),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
bytes_(0),
|
||||
rand_(301) {
|
||||
std::vector<std::string> files;
|
||||
std::string test_dir;
|
||||
Env::Default()->GetTestDirectory(&test_dir);
|
||||
@ -332,7 +330,7 @@ class Benchmark {
|
||||
std::string file_name(test_dir);
|
||||
file_name += "/";
|
||||
file_name += files[i];
|
||||
Env::Default()->RemoveFile(file_name.c_str());
|
||||
Env::Default()->DeleteFile(file_name.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -408,8 +406,7 @@ class Benchmark {
|
||||
} else {
|
||||
known = false;
|
||||
if (name != Slice()) { // No error message for empty name
|
||||
std::fprintf(stderr, "unknown benchmark '%s'\n",
|
||||
name.ToString().c_str());
|
||||
fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
|
||||
}
|
||||
}
|
||||
if (known) {
|
||||
@ -429,26 +426,28 @@ class Benchmark {
|
||||
// Open database
|
||||
std::string tmp_dir;
|
||||
Env::Default()->GetTestDirectory(&tmp_dir);
|
||||
std::snprintf(file_name, sizeof(file_name), "%s/dbbench_sqlite3-%d.db",
|
||||
tmp_dir.c_str(), db_num_);
|
||||
snprintf(file_name, sizeof(file_name),
|
||||
"%s/dbbench_sqlite3-%d.db",
|
||||
tmp_dir.c_str(),
|
||||
db_num_);
|
||||
status = sqlite3_open(file_name, &db_);
|
||||
if (status) {
|
||||
std::fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_));
|
||||
std::exit(1);
|
||||
fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Change SQLite cache size
|
||||
char cache_size[100];
|
||||
std::snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d",
|
||||
FLAGS_num_pages);
|
||||
snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d",
|
||||
FLAGS_num_pages);
|
||||
status = sqlite3_exec(db_, cache_size, nullptr, nullptr, &err_msg);
|
||||
ExecErrorCheck(status, err_msg);
|
||||
|
||||
// FLAGS_page_size is defaulted to 1024
|
||||
if (FLAGS_page_size != 1024) {
|
||||
char page_size[100];
|
||||
std::snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d",
|
||||
FLAGS_page_size);
|
||||
snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d",
|
||||
FLAGS_page_size);
|
||||
status = sqlite3_exec(db_, page_size, nullptr, nullptr, &err_msg);
|
||||
ExecErrorCheck(status, err_msg);
|
||||
}
|
||||
@ -461,27 +460,26 @@ class Benchmark {
|
||||
std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096";
|
||||
status = sqlite3_exec(db_, WAL_stmt.c_str(), nullptr, nullptr, &err_msg);
|
||||
ExecErrorCheck(status, err_msg);
|
||||
status =
|
||||
sqlite3_exec(db_, WAL_checkpoint.c_str(), nullptr, nullptr, &err_msg);
|
||||
status = sqlite3_exec(db_, WAL_checkpoint.c_str(), nullptr, nullptr,
|
||||
&err_msg);
|
||||
ExecErrorCheck(status, err_msg);
|
||||
}
|
||||
|
||||
// Change locking mode to exclusive and create tables/index for database
|
||||
std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE";
|
||||
std::string create_stmt =
|
||||
"CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))";
|
||||
if (!FLAGS_use_rowids) create_stmt += " WITHOUT ROWID";
|
||||
std::string stmt_array[] = {locking_stmt, create_stmt};
|
||||
"CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))";
|
||||
std::string stmt_array[] = { locking_stmt, create_stmt };
|
||||
int stmt_array_length = sizeof(stmt_array) / sizeof(std::string);
|
||||
for (int i = 0; i < stmt_array_length; i++) {
|
||||
status =
|
||||
sqlite3_exec(db_, stmt_array[i].c_str(), nullptr, nullptr, &err_msg);
|
||||
status = sqlite3_exec(db_, stmt_array[i].c_str(), nullptr, nullptr,
|
||||
&err_msg);
|
||||
ExecErrorCheck(status, err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
void Write(bool write_sync, Order order, DBState state, int num_entries,
|
||||
int value_size, int entries_per_batch) {
|
||||
void Write(bool write_sync, Order order, DBState state,
|
||||
int num_entries, int value_size, int entries_per_batch) {
|
||||
// Create new database if state == FRESH
|
||||
if (state == FRESH) {
|
||||
if (FLAGS_use_existing_db) {
|
||||
@ -496,7 +494,7 @@ class Benchmark {
|
||||
|
||||
if (num_entries != num_) {
|
||||
char msg[100];
|
||||
std::snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
|
||||
snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
|
||||
message_ = msg;
|
||||
}
|
||||
|
||||
@ -509,20 +507,20 @@ class Benchmark {
|
||||
std::string end_trans_str = "END TRANSACTION;";
|
||||
|
||||
// Check for synchronous flag in options
|
||||
std::string sync_stmt =
|
||||
(write_sync) ? "PRAGMA synchronous = FULL" : "PRAGMA synchronous = OFF";
|
||||
std::string sync_stmt = (write_sync) ? "PRAGMA synchronous = FULL" :
|
||||
"PRAGMA synchronous = OFF";
|
||||
status = sqlite3_exec(db_, sync_stmt.c_str(), nullptr, nullptr, &err_msg);
|
||||
ExecErrorCheck(status, err_msg);
|
||||
|
||||
// Preparing sqlite3 statements
|
||||
status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1, &replace_stmt,
|
||||
nullptr);
|
||||
status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1,
|
||||
&replace_stmt, nullptr);
|
||||
ErrorCheck(status);
|
||||
status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
|
||||
&begin_trans_stmt, nullptr);
|
||||
ErrorCheck(status);
|
||||
status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt,
|
||||
nullptr);
|
||||
status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1,
|
||||
&end_trans_stmt, nullptr);
|
||||
ErrorCheck(status);
|
||||
|
||||
bool transaction = (entries_per_batch > 1);
|
||||
@ -540,16 +538,16 @@ class Benchmark {
|
||||
const char* value = gen_.Generate(value_size).data();
|
||||
|
||||
// Create values for key-value pair
|
||||
const int k =
|
||||
(order == SEQUENTIAL) ? i + j : (rand_.Next() % num_entries);
|
||||
const int k = (order == SEQUENTIAL) ? i + j :
|
||||
(rand_.Next() % num_entries);
|
||||
char key[100];
|
||||
std::snprintf(key, sizeof(key), "%016d", k);
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
|
||||
// Bind KV values into replace_stmt
|
||||
status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC);
|
||||
ErrorCheck(status);
|
||||
status = sqlite3_bind_blob(replace_stmt, 2, value, value_size,
|
||||
SQLITE_STATIC);
|
||||
status = sqlite3_bind_blob(replace_stmt, 2, value,
|
||||
value_size, SQLITE_STATIC);
|
||||
ErrorCheck(status);
|
||||
|
||||
// Execute replace_stmt
|
||||
@ -595,8 +593,8 @@ class Benchmark {
|
||||
status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
|
||||
&begin_trans_stmt, nullptr);
|
||||
ErrorCheck(status);
|
||||
status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1, &end_trans_stmt,
|
||||
nullptr);
|
||||
status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1,
|
||||
&end_trans_stmt, nullptr);
|
||||
ErrorCheck(status);
|
||||
status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, nullptr);
|
||||
ErrorCheck(status);
|
||||
@ -616,15 +614,14 @@ class Benchmark {
|
||||
// Create key value
|
||||
char key[100];
|
||||
int k = (order == SEQUENTIAL) ? i + j : (rand_.Next() % reads_);
|
||||
std::snprintf(key, sizeof(key), "%016d", k);
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
|
||||
// Bind key value into read_stmt
|
||||
status = sqlite3_bind_blob(read_stmt, 1, key, 16, SQLITE_STATIC);
|
||||
ErrorCheck(status);
|
||||
|
||||
// Execute read statement
|
||||
while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {
|
||||
}
|
||||
while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {}
|
||||
StepErrorCheck(status);
|
||||
|
||||
// Reset SQLite statement for another use
|
||||
@ -654,7 +651,7 @@ class Benchmark {
|
||||
|
||||
void ReadSequential() {
|
||||
int status;
|
||||
sqlite3_stmt* pStmt;
|
||||
sqlite3_stmt *pStmt;
|
||||
std::string read_str = "SELECT * FROM test ORDER BY key";
|
||||
|
||||
status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &pStmt, nullptr);
|
||||
@ -667,6 +664,7 @@ class Benchmark {
|
||||
status = sqlite3_finalize(pStmt);
|
||||
ErrorCheck(status);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
@ -687,9 +685,6 @@ int main(int argc, char** argv) {
|
||||
} else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_use_existing_db = n;
|
||||
} else if (sscanf(argv[i], "--use_rowids=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_use_rowids = n;
|
||||
} else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_num = n;
|
||||
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
|
||||
@ -708,16 +703,16 @@ int main(int argc, char** argv) {
|
||||
} else if (strncmp(argv[i], "--db=", 5) == 0) {
|
||||
FLAGS_db = argv[i] + 5;
|
||||
} else {
|
||||
std::fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
|
||||
std::exit(1);
|
||||
fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Choose a location for the test database if none given with --db=<path>
|
||||
if (FLAGS_db == nullptr) {
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
}
|
||||
|
||||
leveldb::Benchmark benchmark;
|
@ -2,11 +2,9 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <kcpolydb.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "util/histogram.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testutil.h"
|
||||
@ -36,7 +34,8 @@ static const char* FLAGS_benchmarks =
|
||||
"fillrand100K,"
|
||||
"fillseq100K,"
|
||||
"readseq100K,"
|
||||
"readrand100K,";
|
||||
"readrand100K,"
|
||||
;
|
||||
|
||||
// Number of key/values to place in database
|
||||
static int FLAGS_num = 1000000;
|
||||
@ -72,10 +71,12 @@ static bool FLAGS_compression = true;
|
||||
// Use the db with the following name.
|
||||
static const char* FLAGS_db = nullptr;
|
||||
|
||||
inline static void DBSynchronize(kyotocabinet::TreeDB* db_) {
|
||||
inline
|
||||
static void DBSynchronize(kyotocabinet::TreeDB* db_)
|
||||
{
|
||||
// Synchronize will flush writes to disk
|
||||
if (!db_->synchronize()) {
|
||||
std::fprintf(stderr, "synchronize error: %s\n", db_->error().name());
|
||||
fprintf(stderr, "synchronize error: %s\n", db_->error().name());
|
||||
}
|
||||
}
|
||||
|
||||
@ -120,7 +121,7 @@ static Slice TrimSpace(Slice s) {
|
||||
start++;
|
||||
}
|
||||
int limit = s.size();
|
||||
while (limit > start && isspace(s[limit - 1])) {
|
||||
while (limit > start && isspace(s[limit-1])) {
|
||||
limit--;
|
||||
}
|
||||
return Slice(s.data() + start, limit - start);
|
||||
@ -145,52 +146,47 @@ class Benchmark {
|
||||
|
||||
// State kept for progress messages
|
||||
int done_;
|
||||
int next_report_; // When to report next
|
||||
int next_report_; // When to report next
|
||||
|
||||
void PrintHeader() {
|
||||
const int kKeySize = 16;
|
||||
PrintEnvironment();
|
||||
std::fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
|
||||
std::fprintf(
|
||||
stdout, "Values: %d bytes each (%d bytes after compression)\n",
|
||||
FLAGS_value_size,
|
||||
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
|
||||
std::fprintf(stdout, "Entries: %d\n", num_);
|
||||
std::fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
|
||||
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) /
|
||||
1048576.0));
|
||||
std::fprintf(
|
||||
stdout, "FileSize: %.1f MB (estimated)\n",
|
||||
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) /
|
||||
1048576.0));
|
||||
fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
|
||||
fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
|
||||
FLAGS_value_size,
|
||||
static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
|
||||
fprintf(stdout, "Entries: %d\n", num_);
|
||||
fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
|
||||
((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
|
||||
/ 1048576.0));
|
||||
fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
|
||||
(((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
|
||||
/ 1048576.0));
|
||||
PrintWarnings();
|
||||
std::fprintf(stdout, "------------------------------------------------\n");
|
||||
fprintf(stdout, "------------------------------------------------\n");
|
||||
}
|
||||
|
||||
void PrintWarnings() {
|
||||
#if defined(__GNUC__) && !defined(__OPTIMIZE__)
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n");
|
||||
fprintf(stdout,
|
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
|
||||
);
|
||||
#endif
|
||||
#ifndef NDEBUG
|
||||
std::fprintf(
|
||||
stdout,
|
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
|
||||
fprintf(stdout,
|
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
|
||||
#endif
|
||||
}
|
||||
|
||||
void PrintEnvironment() {
|
||||
std::fprintf(
|
||||
stderr, "Kyoto Cabinet: version %s, lib ver %d, lib rev %d\n",
|
||||
kyotocabinet::VERSION, kyotocabinet::LIBVER, kyotocabinet::LIBREV);
|
||||
fprintf(stderr, "Kyoto Cabinet: version %s, lib ver %d, lib rev %d\n",
|
||||
kyotocabinet::VERSION, kyotocabinet::LIBVER, kyotocabinet::LIBREV);
|
||||
|
||||
#if defined(__linux)
|
||||
time_t now = time(nullptr);
|
||||
std::fprintf(stderr, "Date: %s",
|
||||
ctime(&now)); // ctime() adds newline
|
||||
fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
|
||||
|
||||
FILE* cpuinfo = std::fopen("/proc/cpuinfo", "r");
|
||||
FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
|
||||
if (cpuinfo != nullptr) {
|
||||
char line[1000];
|
||||
int num_cpus = 0;
|
||||
@ -210,10 +206,9 @@ class Benchmark {
|
||||
cache_size = val.ToString();
|
||||
}
|
||||
}
|
||||
std::fclose(cpuinfo);
|
||||
std::fprintf(stderr, "CPU: %d * %s\n", num_cpus,
|
||||
cpu_type.c_str());
|
||||
std::fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
|
||||
fclose(cpuinfo);
|
||||
fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
|
||||
fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@ -234,30 +229,23 @@ class Benchmark {
|
||||
double micros = (now - last_op_finish_) * 1e6;
|
||||
hist_.Add(micros);
|
||||
if (micros > 20000) {
|
||||
std::fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
|
||||
std::fflush(stderr);
|
||||
fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
|
||||
fflush(stderr);
|
||||
}
|
||||
last_op_finish_ = now;
|
||||
}
|
||||
|
||||
done_++;
|
||||
if (done_ >= next_report_) {
|
||||
if (next_report_ < 1000)
|
||||
next_report_ += 100;
|
||||
else if (next_report_ < 5000)
|
||||
next_report_ += 500;
|
||||
else if (next_report_ < 10000)
|
||||
next_report_ += 1000;
|
||||
else if (next_report_ < 50000)
|
||||
next_report_ += 5000;
|
||||
else if (next_report_ < 100000)
|
||||
next_report_ += 10000;
|
||||
else if (next_report_ < 500000)
|
||||
next_report_ += 50000;
|
||||
else
|
||||
next_report_ += 100000;
|
||||
std::fprintf(stderr, "... finished %d ops%30s\r", done_, "");
|
||||
std::fflush(stderr);
|
||||
if (next_report_ < 1000) next_report_ += 100;
|
||||
else if (next_report_ < 5000) next_report_ += 500;
|
||||
else if (next_report_ < 10000) next_report_ += 1000;
|
||||
else if (next_report_ < 50000) next_report_ += 5000;
|
||||
else if (next_report_ < 100000) next_report_ += 10000;
|
||||
else if (next_report_ < 500000) next_report_ += 50000;
|
||||
else next_report_ += 100000;
|
||||
fprintf(stderr, "... finished %d ops%30s\r", done_, "");
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
|
||||
@ -270,35 +258,42 @@ class Benchmark {
|
||||
|
||||
if (bytes_ > 0) {
|
||||
char rate[100];
|
||||
std::snprintf(rate, sizeof(rate), "%6.1f MB/s",
|
||||
(bytes_ / 1048576.0) / (finish - start_));
|
||||
snprintf(rate, sizeof(rate), "%6.1f MB/s",
|
||||
(bytes_ / 1048576.0) / (finish - start_));
|
||||
if (!message_.empty()) {
|
||||
message_ = std::string(rate) + " " + message_;
|
||||
message_ = std::string(rate) + " " + message_;
|
||||
} else {
|
||||
message_ = rate;
|
||||
}
|
||||
}
|
||||
|
||||
std::fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
|
||||
name.ToString().c_str(), (finish - start_) * 1e6 / done_,
|
||||
(message_.empty() ? "" : " "), message_.c_str());
|
||||
fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
|
||||
name.ToString().c_str(),
|
||||
(finish - start_) * 1e6 / done_,
|
||||
(message_.empty() ? "" : " "),
|
||||
message_.c_str());
|
||||
if (FLAGS_histogram) {
|
||||
std::fprintf(stdout, "Microseconds per op:\n%s\n",
|
||||
hist_.ToString().c_str());
|
||||
fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
|
||||
}
|
||||
std::fflush(stdout);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
public:
|
||||
enum Order { SEQUENTIAL, RANDOM };
|
||||
enum DBState { FRESH, EXISTING };
|
||||
enum Order {
|
||||
SEQUENTIAL,
|
||||
RANDOM
|
||||
};
|
||||
enum DBState {
|
||||
FRESH,
|
||||
EXISTING
|
||||
};
|
||||
|
||||
Benchmark()
|
||||
: db_(nullptr),
|
||||
num_(FLAGS_num),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
bytes_(0),
|
||||
rand_(301) {
|
||||
: db_(nullptr),
|
||||
num_(FLAGS_num),
|
||||
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
|
||||
bytes_(0),
|
||||
rand_(301) {
|
||||
std::vector<std::string> files;
|
||||
std::string test_dir;
|
||||
Env::Default()->GetTestDirectory(&test_dir);
|
||||
@ -309,7 +304,7 @@ class Benchmark {
|
||||
std::string file_name(test_dir);
|
||||
file_name += "/";
|
||||
file_name += files[i];
|
||||
Env::Default()->RemoveFile(file_name.c_str());
|
||||
Env::Default()->DeleteFile(file_name.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -317,7 +312,7 @@ class Benchmark {
|
||||
|
||||
~Benchmark() {
|
||||
if (!db_->close()) {
|
||||
std::fprintf(stderr, "close error: %s\n", db_->error().name());
|
||||
fprintf(stderr, "close error: %s\n", db_->error().name());
|
||||
}
|
||||
}
|
||||
|
||||
@ -381,8 +376,7 @@ class Benchmark {
|
||||
} else {
|
||||
known = false;
|
||||
if (name != Slice()) { // No error message for empty name
|
||||
std::fprintf(stderr, "unknown benchmark '%s'\n",
|
||||
name.ToString().c_str());
|
||||
fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
|
||||
}
|
||||
}
|
||||
if (known) {
|
||||
@ -392,7 +386,7 @@ class Benchmark {
|
||||
}
|
||||
|
||||
private:
|
||||
void Open(bool sync) {
|
||||
void Open(bool sync) {
|
||||
assert(db_ == nullptr);
|
||||
|
||||
// Initialize db_
|
||||
@ -401,14 +395,16 @@ class Benchmark {
|
||||
db_num_++;
|
||||
std::string test_dir;
|
||||
Env::Default()->GetTestDirectory(&test_dir);
|
||||
std::snprintf(file_name, sizeof(file_name), "%s/dbbench_polyDB-%d.kct",
|
||||
test_dir.c_str(), db_num_);
|
||||
snprintf(file_name, sizeof(file_name),
|
||||
"%s/dbbench_polyDB-%d.kct",
|
||||
test_dir.c_str(),
|
||||
db_num_);
|
||||
|
||||
// Create tuning options and open the database
|
||||
int open_options =
|
||||
kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE;
|
||||
int tune_options =
|
||||
kyotocabinet::TreeDB::TSMALL | kyotocabinet::TreeDB::TLINEAR;
|
||||
int open_options = kyotocabinet::PolyDB::OWRITER |
|
||||
kyotocabinet::PolyDB::OCREATE;
|
||||
int tune_options = kyotocabinet::TreeDB::TSMALL |
|
||||
kyotocabinet::TreeDB::TLINEAR;
|
||||
if (FLAGS_compression) {
|
||||
tune_options |= kyotocabinet::TreeDB::TCOMPRESS;
|
||||
db_->tune_compressor(&comp_);
|
||||
@ -416,17 +412,17 @@ class Benchmark {
|
||||
db_->tune_options(tune_options);
|
||||
db_->tune_page_cache(FLAGS_cache_size);
|
||||
db_->tune_page(FLAGS_page_size);
|
||||
db_->tune_map(256LL << 20);
|
||||
db_->tune_map(256LL<<20);
|
||||
if (sync) {
|
||||
open_options |= kyotocabinet::PolyDB::OAUTOSYNC;
|
||||
}
|
||||
if (!db_->open(file_name, open_options)) {
|
||||
std::fprintf(stderr, "open error: %s\n", db_->error().name());
|
||||
fprintf(stderr, "open error: %s\n", db_->error().name());
|
||||
}
|
||||
}
|
||||
|
||||
void Write(bool sync, Order order, DBState state, int num_entries,
|
||||
int value_size, int entries_per_batch) {
|
||||
void Write(bool sync, Order order, DBState state,
|
||||
int num_entries, int value_size, int entries_per_batch) {
|
||||
// Create new database if state == FRESH
|
||||
if (state == FRESH) {
|
||||
if (FLAGS_use_existing_db) {
|
||||
@ -441,19 +437,20 @@ class Benchmark {
|
||||
|
||||
if (num_entries != num_) {
|
||||
char msg[100];
|
||||
std::snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
|
||||
snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
|
||||
message_ = msg;
|
||||
}
|
||||
|
||||
// Write to database
|
||||
for (int i = 0; i < num_entries; i++) {
|
||||
for (int i = 0; i < num_entries; i++)
|
||||
{
|
||||
const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries);
|
||||
char key[100];
|
||||
std::snprintf(key, sizeof(key), "%016d", k);
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
bytes_ += value_size + strlen(key);
|
||||
std::string cpp_key = key;
|
||||
if (!db_->set(cpp_key, gen_.Generate(value_size).ToString())) {
|
||||
std::fprintf(stderr, "set error: %s\n", db_->error().name());
|
||||
fprintf(stderr, "set error: %s\n", db_->error().name());
|
||||
}
|
||||
FinishedSingleOp();
|
||||
}
|
||||
@ -475,7 +472,7 @@ class Benchmark {
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
char key[100];
|
||||
const int k = rand_.Next() % reads_;
|
||||
std::snprintf(key, sizeof(key), "%016d", k);
|
||||
snprintf(key, sizeof(key), "%016d", k);
|
||||
db_->get(key, &value);
|
||||
FinishedSingleOp();
|
||||
}
|
||||
@ -513,16 +510,16 @@ int main(int argc, char** argv) {
|
||||
} else if (strncmp(argv[i], "--db=", 5) == 0) {
|
||||
FLAGS_db = argv[i] + 5;
|
||||
} else {
|
||||
std::fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
|
||||
std::exit(1);
|
||||
fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Choose a location for the test database if none given with --db=<path>
|
||||
if (FLAGS_db == nullptr) {
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
}
|
||||
|
||||
leveldb::Benchmark benchmark;
|
@ -83,23 +83,23 @@ div.bsql {
|
||||
<p>Google, July 2011</p>
|
||||
<hr>
|
||||
|
||||
<p>In order to test LevelDB's performance, we benchmark it against other well-established database implementations. We compare LevelDB (revision 39) against <a href="https://www.sqlite.org/">SQLite3</a> (version 3.7.6.3) and <a href="https://dbmx.net/kyotocabinet/spex.html">Kyoto Cabinet's</a> (version 1.2.67) TreeDB (a B+Tree based key-value store). We would like to acknowledge Scott Hess and Mikio Hirabayashi for their suggestions and contributions to the SQLite3 and Kyoto Cabinet benchmarks, respectively.</p>
|
||||
<p>In order to test LevelDB's performance, we benchmark it against other well-established database implementations. We compare LevelDB (revision 39) against <a href="http://www.sqlite.org/">SQLite3</a> (version 3.7.6.3) and <a href="http://fallabs.com/kyotocabinet/spex.html">Kyoto Cabinet's</a> (version 1.2.67) TreeDB (a B+Tree based key-value store). We would like to acknowledge Scott Hess and Mikio Hirabayashi for their suggestions and contributions to the SQLite3 and Kyoto Cabinet benchmarks, respectively.</p>
|
||||
|
||||
<p>Benchmarks were all performed on a six-core Intel(R) Xeon(R) CPU X5650 @ 2.67GHz, with 12288 KB of total L3 cache and 12 GB of DDR3 RAM at 1333 MHz. (Note that LevelDB uses at most two CPUs since the benchmarks are single threaded: one to run the benchmark, and one for background compactions.) We ran the benchmarks on two machines (with identical processors), one with an Ext3 file system and one with an Ext4 file system. The machine with the Ext3 file system has a SATA Hitachi HDS721050CLA362 hard drive. The machine with the Ext4 file system has a SATA Samsung HD502HJ hard drive. Both hard drives spin at 7200 RPM and have hard drive write-caching enabled (using `hdparm -W 1 [device]`). The numbers reported below are the median of three measurements.</p>
|
||||
|
||||
<h4>Benchmark Source Code</h4>
|
||||
<p>We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's <span class="code">db_bench</span>. The code for each of the benchmarks resides here:</p>
|
||||
<ul>
|
||||
<li> <b>LevelDB:</b> <a href="https://github.com/google/leveldb/blob/main/benchmarks/db_bench.cc">benchmarks/db_bench.cc</a>.</li>
|
||||
<li> <b>SQLite:</b> <a href="https://github.com/google/leveldb/blob/main/benchmarks/db_bench_sqlite3.cc">benchmarks/db_bench_sqlite3.cc</a>.</li>
|
||||
<li> <b>Kyoto TreeDB:</b> <a href="https://github.com/google/leveldb/blob/main/benchmarks/db_bench_tree_db.cc">benchmarks/db_bench_tree_db.cc</a>.</li>
|
||||
<li> <b>LevelDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/trunk/db/db_bench.cc">db/db_bench.cc</a>.</li>
|
||||
<li> <b>SQLite:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_sqlite3.cc">doc/bench/db_bench_sqlite3.cc</a>.</li>
|
||||
<li> <b>Kyoto TreeDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_tree_db.cc">doc/bench/db_bench_tree_db.cc</a>.</li>
|
||||
</ul>
|
||||
|
||||
<h4>Custom Build Specifications</h4>
|
||||
<ul>
|
||||
<li>LevelDB: LevelDB was compiled with the <a href="https://github.com/gperftools/gperftools">tcmalloc</a> library and the <a href="https://github.com/google/snappy">Snappy</a> compression library (revision 33). Assertions were disabled.</li>
|
||||
<li>TreeDB: TreeDB was compiled using the <a href="https://www.oberhumer.com/opensource/lzo/">LZO</a> compression library (version 2.03). Furthermore, we enabled the TSMALL and TLINEAR options when opening the database in order to reduce the footprint of each record.</li>
|
||||
<li>SQLite: We tuned SQLite's performance, by setting its locking mode to exclusive. We also enabled SQLite's <a href="https://www.sqlite.org/draft/wal.html">write-ahead logging</a>.</li>
|
||||
<li>LevelDB: LevelDB was compiled with the <a href="http://code.google.com/p/google-perftools">tcmalloc</a> library and the <a href="http://code.google.com/p/snappy/">Snappy</a> compression library (revision 33). Assertions were disabled.</li>
|
||||
<li>TreeDB: TreeDB was compiled using the <a href="http://www.oberhumer.com/opensource/lzo/">LZO</a> compression library (version 2.03). Furthermore, we enabled the TSMALL and TLINEAR options when opening the database in order to reduce the footprint of each record.</li>
|
||||
<li>SQLite: We tuned SQLite's performance, by setting its locking mode to exclusive. We also enabled SQLite's <a href="http://www.sqlite.org/draft/wal.html">write-ahead logging</a>.</li>
|
||||
</ul>
|
||||
|
||||
<h2>1. Baseline Performance</h2>
|
||||
@ -451,7 +451,7 @@ performance may very well be better with compression if it allows more
|
||||
of the working set to fit in memory.</p>
|
||||
|
||||
<h2>Note about Ext4 Filesystems</h2>
|
||||
<p>The preceding numbers are for an ext3 file system. Synchronous writes are much slower under <a href="https://en.wikipedia.org/wiki/Ext4">ext4</a> (LevelDB drops to ~31 writes / second and TreeDB drops to ~5 writes / second; SQLite3's synchronous writes do not noticeably drop) due to ext4's different handling of <span class="code">fsync</span> / <span class="code">msync</span> calls. Even LevelDB's asynchronous write performance drops somewhat since it spreads its storage across multiple files and issues <span class="code">fsync</span> calls when switching to a new file.</p>
|
||||
<p>The preceding numbers are for an ext3 file system. Synchronous writes are much slower under <a href="http://en.wikipedia.org/wiki/Ext4">ext4</a> (LevelDB drops to ~31 writes / second and TreeDB drops to ~5 writes / second; SQLite3's synchronous writes do not noticeably drop) due to ext4's different handling of <span class="code">fsync</span> / <span class="code">msync</span> calls. Even LevelDB's asynchronous write performance drops somewhat since it spreads its storage across multiple files and issues <span class="code">fsync</span> calls when switching to a new file.</p>
|
||||
|
||||
<h2>Acknowledgements</h2>
|
||||
<p>Jeff Dean and Sanjay Ghemawat wrote LevelDB. Kevin Tseng wrote and compiled these benchmarks. Mikio Hirabayashi, Scott Hess, and Gabor Cselle provided help and advice.</p>
|
||||
|
@ -1,7 +1,7 @@
|
||||
## Files
|
||||
|
||||
The implementation of leveldb is similar in spirit to the representation of a
|
||||
single [Bigtable tablet (section 5.3)](https://research.google/pubs/pub27898/).
|
||||
single [Bigtable tablet (section 5.3)](http://research.google.com/archive/bigtable.html).
|
||||
However the organization of the files that make up the representation is
|
||||
somewhat different and is explained below.
|
||||
|
||||
@ -166,7 +166,7 @@ So maybe even the sharding is not necessary on modern filesystems?
|
||||
|
||||
## Garbage collection of files
|
||||
|
||||
`RemoveObsoleteFiles()` is called at the end of every compaction and at the end
|
||||
`DeleteObsoleteFiles()` is called at the end of every compaction and at the end
|
||||
of recovery. It finds the names of all files in the database. It deletes all log
|
||||
files that are not the current log file. It deletes all table files that are not
|
||||
referenced from some level and are not the output of an active compaction.
|
||||
|
15
doc/index.md
15
doc/index.md
@ -307,7 +307,7 @@ version numbers found in the keys to decide how to interpret them.
|
||||
## Performance
|
||||
|
||||
Performance can be tuned by changing the default values of the types defined in
|
||||
`include/options.h`.
|
||||
`include/leveldb/options.h`.
|
||||
|
||||
### Block size
|
||||
|
||||
@ -369,7 +369,6 @@ leveldb::Iterator* it = db->NewIterator(options);
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
...
|
||||
}
|
||||
delete it;
|
||||
```
|
||||
|
||||
### Key Layout
|
||||
@ -425,21 +424,21 @@ spaces. For example:
|
||||
```c++
|
||||
class CustomFilterPolicy : public leveldb::FilterPolicy {
|
||||
private:
|
||||
leveldb::FilterPolicy* builtin_policy_;
|
||||
FilterPolicy* builtin_policy_;
|
||||
|
||||
public:
|
||||
CustomFilterPolicy() : builtin_policy_(leveldb::NewBloomFilterPolicy(10)) {}
|
||||
CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) {}
|
||||
~CustomFilterPolicy() { delete builtin_policy_; }
|
||||
|
||||
const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
|
||||
|
||||
void CreateFilter(const leveldb::Slice* keys, int n, std::string* dst) const {
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
// Use builtin bloom filter code after removing trailing spaces
|
||||
std::vector<leveldb::Slice> trimmed(n);
|
||||
std::vector<Slice> trimmed(n);
|
||||
for (int i = 0; i < n; i++) {
|
||||
trimmed[i] = RemoveTrailingSpaces(keys[i]);
|
||||
}
|
||||
builtin_policy_->CreateFilter(trimmed.data(), n, dst);
|
||||
return builtin_policy_->CreateFilter(&trimmed[i], n, dst);
|
||||
}
|
||||
};
|
||||
```
|
||||
@ -479,7 +478,7 @@ leveldb::Range ranges[2];
|
||||
ranges[0] = leveldb::Range("a", "c");
|
||||
ranges[1] = leveldb::Range("x", "z");
|
||||
uint64_t sizes[2];
|
||||
db->GetApproximateSizes(ranges, 2, sizes);
|
||||
leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
|
||||
```
|
||||
|
||||
The preceding call will set `sizes[0]` to the approximate number of bytes of
|
||||
|
@ -4,7 +4,8 @@
|
||||
|
||||
#include "helpers/memenv/memenv.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <string.h>
|
||||
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <string>
|
||||
@ -26,10 +27,6 @@ class FileState {
|
||||
// and the caller must call Ref() at least once.
|
||||
FileState() : refs_(0), size_(0) {}
|
||||
|
||||
// No copying allowed.
|
||||
FileState(const FileState&) = delete;
|
||||
FileState& operator=(const FileState&) = delete;
|
||||
|
||||
// Increase the reference count.
|
||||
void Ref() {
|
||||
MutexLock lock(&refs_mutex_);
|
||||
@ -93,7 +90,7 @@ class FileState {
|
||||
if (avail > bytes_to_copy) {
|
||||
avail = bytes_to_copy;
|
||||
}
|
||||
std::memcpy(dst, blocks_[block] + block_offset, avail);
|
||||
memcpy(dst, blocks_[block] + block_offset, avail);
|
||||
|
||||
bytes_to_copy -= avail;
|
||||
dst += avail;
|
||||
@ -126,7 +123,7 @@ class FileState {
|
||||
if (avail > src_len) {
|
||||
avail = src_len;
|
||||
}
|
||||
std::memcpy(blocks_.back() + offset, src, avail);
|
||||
memcpy(blocks_.back() + offset, src, avail);
|
||||
src_len -= avail;
|
||||
src += avail;
|
||||
size_ += avail;
|
||||
@ -136,10 +133,14 @@ class FileState {
|
||||
}
|
||||
|
||||
private:
|
||||
enum { kBlockSize = 8 * 1024 };
|
||||
|
||||
// Private since only Unref() should be used to delete it.
|
||||
~FileState() { Truncate(); }
|
||||
~FileState() {
|
||||
Truncate();
|
||||
}
|
||||
|
||||
// No copying allowed.
|
||||
FileState(const FileState&);
|
||||
void operator=(const FileState&);
|
||||
|
||||
port::Mutex refs_mutex_;
|
||||
int refs_ GUARDED_BY(refs_mutex_);
|
||||
@ -147,6 +148,8 @@ class FileState {
|
||||
mutable port::Mutex blocks_mutex_;
|
||||
std::vector<char*> blocks_ GUARDED_BY(blocks_mutex_);
|
||||
uint64_t size_ GUARDED_BY(blocks_mutex_);
|
||||
|
||||
enum { kBlockSize = 8 * 1024 };
|
||||
};
|
||||
|
||||
class SequentialFileImpl : public SequentialFile {
|
||||
@ -155,9 +158,11 @@ class SequentialFileImpl : public SequentialFile {
|
||||
file_->Ref();
|
||||
}
|
||||
|
||||
~SequentialFileImpl() override { file_->Unref(); }
|
||||
~SequentialFileImpl() {
|
||||
file_->Unref();
|
||||
}
|
||||
|
||||
Status Read(size_t n, Slice* result, char* scratch) override {
|
||||
virtual Status Read(size_t n, Slice* result, char* scratch) {
|
||||
Status s = file_->Read(pos_, n, result, scratch);
|
||||
if (s.ok()) {
|
||||
pos_ += result->size();
|
||||
@ -165,7 +170,7 @@ class SequentialFileImpl : public SequentialFile {
|
||||
return s;
|
||||
}
|
||||
|
||||
Status Skip(uint64_t n) override {
|
||||
virtual Status Skip(uint64_t n) {
|
||||
if (pos_ > file_->Size()) {
|
||||
return Status::IOError("pos_ > file_->Size()");
|
||||
}
|
||||
@ -184,12 +189,16 @@ class SequentialFileImpl : public SequentialFile {
|
||||
|
||||
class RandomAccessFileImpl : public RandomAccessFile {
|
||||
public:
|
||||
explicit RandomAccessFileImpl(FileState* file) : file_(file) { file_->Ref(); }
|
||||
explicit RandomAccessFileImpl(FileState* file) : file_(file) {
|
||||
file_->Ref();
|
||||
}
|
||||
|
||||
~RandomAccessFileImpl() override { file_->Unref(); }
|
||||
~RandomAccessFileImpl() {
|
||||
file_->Unref();
|
||||
}
|
||||
|
||||
Status Read(uint64_t offset, size_t n, Slice* result,
|
||||
char* scratch) const override {
|
||||
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
||||
char* scratch) const {
|
||||
return file_->Read(offset, n, result, scratch);
|
||||
}
|
||||
|
||||
@ -199,15 +208,21 @@ class RandomAccessFileImpl : public RandomAccessFile {
|
||||
|
||||
class WritableFileImpl : public WritableFile {
|
||||
public:
|
||||
WritableFileImpl(FileState* file) : file_(file) { file_->Ref(); }
|
||||
WritableFileImpl(FileState* file) : file_(file) {
|
||||
file_->Ref();
|
||||
}
|
||||
|
||||
~WritableFileImpl() override { file_->Unref(); }
|
||||
~WritableFileImpl() {
|
||||
file_->Unref();
|
||||
}
|
||||
|
||||
Status Append(const Slice& data) override { return file_->Append(data); }
|
||||
virtual Status Append(const Slice& data) {
|
||||
return file_->Append(data);
|
||||
}
|
||||
|
||||
Status Close() override { return Status::OK(); }
|
||||
Status Flush() override { return Status::OK(); }
|
||||
Status Sync() override { return Status::OK(); }
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
|
||||
private:
|
||||
FileState* file_;
|
||||
@ -215,22 +230,22 @@ class WritableFileImpl : public WritableFile {
|
||||
|
||||
class NoOpLogger : public Logger {
|
||||
public:
|
||||
void Logv(const char* format, std::va_list ap) override {}
|
||||
virtual void Logv(const char* format, va_list ap) { }
|
||||
};
|
||||
|
||||
class InMemoryEnv : public EnvWrapper {
|
||||
public:
|
||||
explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) {}
|
||||
explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
|
||||
|
||||
~InMemoryEnv() override {
|
||||
for (const auto& kvp : file_map_) {
|
||||
kvp.second->Unref();
|
||||
virtual ~InMemoryEnv() {
|
||||
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
|
||||
i->second->Unref();
|
||||
}
|
||||
}
|
||||
|
||||
// Partial implementation of the Env interface.
|
||||
Status NewSequentialFile(const std::string& fname,
|
||||
SequentialFile** result) override {
|
||||
virtual Status NewSequentialFile(const std::string& fname,
|
||||
SequentialFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
*result = nullptr;
|
||||
@ -241,8 +256,8 @@ class InMemoryEnv : public EnvWrapper {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status NewRandomAccessFile(const std::string& fname,
|
||||
RandomAccessFile** result) override {
|
||||
virtual Status NewRandomAccessFile(const std::string& fname,
|
||||
RandomAccessFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
*result = nullptr;
|
||||
@ -253,8 +268,8 @@ class InMemoryEnv : public EnvWrapper {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result) override {
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
FileSystem::iterator it = file_map_.find(fname);
|
||||
|
||||
@ -273,8 +288,8 @@ class InMemoryEnv : public EnvWrapper {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status NewAppendableFile(const std::string& fname,
|
||||
WritableFile** result) override {
|
||||
virtual Status NewAppendableFile(const std::string& fname,
|
||||
WritableFile** result) {
|
||||
MutexLock lock(&mutex_);
|
||||
FileState** sptr = &file_map_[fname];
|
||||
FileState* file = *sptr;
|
||||
@ -286,18 +301,18 @@ class InMemoryEnv : public EnvWrapper {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool FileExists(const std::string& fname) override {
|
||||
virtual bool FileExists(const std::string& fname) {
|
||||
MutexLock lock(&mutex_);
|
||||
return file_map_.find(fname) != file_map_.end();
|
||||
}
|
||||
|
||||
Status GetChildren(const std::string& dir,
|
||||
std::vector<std::string>* result) override {
|
||||
virtual Status GetChildren(const std::string& dir,
|
||||
std::vector<std::string>* result) {
|
||||
MutexLock lock(&mutex_);
|
||||
result->clear();
|
||||
|
||||
for (const auto& kvp : file_map_) {
|
||||
const std::string& filename = kvp.first;
|
||||
for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
|
||||
const std::string& filename = i->first;
|
||||
|
||||
if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
|
||||
Slice(filename).starts_with(Slice(dir))) {
|
||||
@ -308,7 +323,7 @@ class InMemoryEnv : public EnvWrapper {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void RemoveFileInternal(const std::string& fname)
|
||||
void DeleteFileInternal(const std::string& fname)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
return;
|
||||
@ -318,21 +333,25 @@ class InMemoryEnv : public EnvWrapper {
|
||||
file_map_.erase(fname);
|
||||
}
|
||||
|
||||
Status RemoveFile(const std::string& fname) override {
|
||||
virtual Status DeleteFile(const std::string& fname) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
return Status::IOError(fname, "File not found");
|
||||
}
|
||||
|
||||
RemoveFileInternal(fname);
|
||||
DeleteFileInternal(fname);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status CreateDir(const std::string& dirname) override { return Status::OK(); }
|
||||
virtual Status CreateDir(const std::string& dirname) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RemoveDir(const std::string& dirname) override { return Status::OK(); }
|
||||
virtual Status DeleteDir(const std::string& dirname) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status GetFileSize(const std::string& fname, uint64_t* file_size) override {
|
||||
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
return Status::IOError(fname, "File not found");
|
||||
@ -342,35 +361,35 @@ class InMemoryEnv : public EnvWrapper {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RenameFile(const std::string& src,
|
||||
const std::string& target) override {
|
||||
virtual Status RenameFile(const std::string& src,
|
||||
const std::string& target) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(src) == file_map_.end()) {
|
||||
return Status::IOError(src, "File not found");
|
||||
}
|
||||
|
||||
RemoveFileInternal(target);
|
||||
DeleteFileInternal(target);
|
||||
file_map_[target] = file_map_[src];
|
||||
file_map_.erase(src);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status LockFile(const std::string& fname, FileLock** lock) override {
|
||||
virtual Status LockFile(const std::string& fname, FileLock** lock) {
|
||||
*lock = new FileLock;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status UnlockFile(FileLock* lock) override {
|
||||
virtual Status UnlockFile(FileLock* lock) {
|
||||
delete lock;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status GetTestDirectory(std::string* path) override {
|
||||
virtual Status GetTestDirectory(std::string* path) {
|
||||
*path = "/test";
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status NewLogger(const std::string& fname, Logger** result) override {
|
||||
virtual Status NewLogger(const std::string& fname, Logger** result) {
|
||||
*result = new NoOpLogger;
|
||||
return Status::OK();
|
||||
}
|
||||
@ -378,13 +397,14 @@ class InMemoryEnv : public EnvWrapper {
|
||||
private:
|
||||
// Map from filenames to FileState objects, representing a simple file system.
|
||||
typedef std::map<std::string, FileState*> FileSystem;
|
||||
|
||||
port::Mutex mutex_;
|
||||
FileSystem file_map_ GUARDED_BY(mutex_);
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
Env* NewMemEnv(Env* base_env) { return new InMemoryEnv(base_env); }
|
||||
Env* NewMemEnv(Env* base_env) {
|
||||
return new InMemoryEnv(base_env);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -4,74 +4,76 @@
|
||||
|
||||
#include "helpers/memenv/memenv.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/testutil.h"
|
||||
#include "util/testharness.h"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class MemEnvTest : public testing::Test {
|
||||
class MemEnvTest {
|
||||
public:
|
||||
MemEnvTest() : env_(NewMemEnv(Env::Default())) {}
|
||||
~MemEnvTest() { delete env_; }
|
||||
|
||||
Env* env_;
|
||||
|
||||
MemEnvTest()
|
||||
: env_(NewMemEnv(Env::Default())) {
|
||||
}
|
||||
~MemEnvTest() {
|
||||
delete env_;
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(MemEnvTest, Basics) {
|
||||
TEST(MemEnvTest, Basics) {
|
||||
uint64_t file_size;
|
||||
WritableFile* writable_file;
|
||||
std::vector<std::string> children;
|
||||
|
||||
ASSERT_LEVELDB_OK(env_->CreateDir("/dir"));
|
||||
ASSERT_OK(env_->CreateDir("/dir"));
|
||||
|
||||
// Check that the directory is empty.
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
|
||||
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
|
||||
ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_EQ(0, children.size());
|
||||
|
||||
// Create a file.
|
||||
ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_EQ(0, file_size);
|
||||
delete writable_file;
|
||||
|
||||
// Check that the file exists.
|
||||
ASSERT_TRUE(env_->FileExists("/dir/f"));
|
||||
ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_EQ(0, file_size);
|
||||
ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_EQ(1, children.size());
|
||||
ASSERT_EQ("f", children[0]);
|
||||
|
||||
// Write to the file.
|
||||
ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_LEVELDB_OK(writable_file->Append("abc"));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("abc"));
|
||||
delete writable_file;
|
||||
|
||||
// Check that append works.
|
||||
ASSERT_LEVELDB_OK(env_->NewAppendableFile("/dir/f", &writable_file));
|
||||
ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_OK(env_->NewAppendableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_EQ(3, file_size);
|
||||
ASSERT_LEVELDB_OK(writable_file->Append("hello"));
|
||||
ASSERT_OK(writable_file->Append("hello"));
|
||||
delete writable_file;
|
||||
|
||||
// Check for expected size.
|
||||
ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
|
||||
ASSERT_EQ(8, file_size);
|
||||
|
||||
// Check that renaming works.
|
||||
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
|
||||
ASSERT_LEVELDB_OK(env_->RenameFile("/dir/f", "/dir/g"));
|
||||
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/f"));
|
||||
ASSERT_TRUE(env_->FileExists("/dir/g"));
|
||||
ASSERT_LEVELDB_OK(env_->GetFileSize("/dir/g", &file_size));
|
||||
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
|
||||
ASSERT_EQ(8, file_size);
|
||||
|
||||
// Check that opening non-existent file fails.
|
||||
@ -83,50 +85,49 @@ TEST_F(MemEnvTest, Basics) {
|
||||
ASSERT_TRUE(!rand_file);
|
||||
|
||||
// Check that deleting works.
|
||||
ASSERT_TRUE(!env_->RemoveFile("/dir/non_existent").ok());
|
||||
ASSERT_LEVELDB_OK(env_->RemoveFile("/dir/g"));
|
||||
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
|
||||
ASSERT_OK(env_->DeleteFile("/dir/g"));
|
||||
ASSERT_TRUE(!env_->FileExists("/dir/g"));
|
||||
ASSERT_LEVELDB_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_OK(env_->GetChildren("/dir", &children));
|
||||
ASSERT_EQ(0, children.size());
|
||||
ASSERT_LEVELDB_OK(env_->RemoveDir("/dir"));
|
||||
ASSERT_OK(env_->DeleteDir("/dir"));
|
||||
}
|
||||
|
||||
TEST_F(MemEnvTest, ReadWrite) {
|
||||
TEST(MemEnvTest, ReadWrite) {
|
||||
WritableFile* writable_file;
|
||||
SequentialFile* seq_file;
|
||||
RandomAccessFile* rand_file;
|
||||
Slice result;
|
||||
char scratch[100];
|
||||
|
||||
ASSERT_LEVELDB_OK(env_->CreateDir("/dir"));
|
||||
ASSERT_OK(env_->CreateDir("/dir"));
|
||||
|
||||
ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_LEVELDB_OK(writable_file->Append("hello "));
|
||||
ASSERT_LEVELDB_OK(writable_file->Append("world"));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("hello "));
|
||||
ASSERT_OK(writable_file->Append("world"));
|
||||
delete writable_file;
|
||||
|
||||
// Read sequentially.
|
||||
ASSERT_LEVELDB_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_LEVELDB_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello"));
|
||||
ASSERT_LEVELDB_OK(seq_file->Skip(1));
|
||||
ASSERT_LEVELDB_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
|
||||
ASSERT_OK(seq_file->Skip(1));
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world"));
|
||||
ASSERT_LEVELDB_OK(
|
||||
seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
|
||||
ASSERT_EQ(0, result.size());
|
||||
ASSERT_LEVELDB_OK(seq_file->Skip(100)); // Try to skip past end of file.
|
||||
ASSERT_LEVELDB_OK(seq_file->Read(1000, &result, scratch));
|
||||
ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
|
||||
ASSERT_OK(seq_file->Read(1000, &result, scratch));
|
||||
ASSERT_EQ(0, result.size());
|
||||
delete seq_file;
|
||||
|
||||
// Random reads.
|
||||
ASSERT_LEVELDB_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
|
||||
ASSERT_LEVELDB_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
|
||||
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
|
||||
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world"));
|
||||
ASSERT_LEVELDB_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
|
||||
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello"));
|
||||
ASSERT_LEVELDB_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
|
||||
ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
|
||||
ASSERT_EQ(0, result.compare("d"));
|
||||
|
||||
// Too high offset.
|
||||
@ -134,30 +135,30 @@ TEST_F(MemEnvTest, ReadWrite) {
|
||||
delete rand_file;
|
||||
}
|
||||
|
||||
TEST_F(MemEnvTest, Locks) {
|
||||
TEST(MemEnvTest, Locks) {
|
||||
FileLock* lock;
|
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_LEVELDB_OK(env_->LockFile("some file", &lock));
|
||||
ASSERT_LEVELDB_OK(env_->UnlockFile(lock));
|
||||
ASSERT_OK(env_->LockFile("some file", &lock));
|
||||
ASSERT_OK(env_->UnlockFile(lock));
|
||||
}
|
||||
|
||||
TEST_F(MemEnvTest, Misc) {
|
||||
TEST(MemEnvTest, Misc) {
|
||||
std::string test_dir;
|
||||
ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir));
|
||||
ASSERT_OK(env_->GetTestDirectory(&test_dir));
|
||||
ASSERT_TRUE(!test_dir.empty());
|
||||
|
||||
WritableFile* writable_file;
|
||||
ASSERT_LEVELDB_OK(env_->NewWritableFile("/a/b", &writable_file));
|
||||
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
|
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_LEVELDB_OK(writable_file->Sync());
|
||||
ASSERT_LEVELDB_OK(writable_file->Flush());
|
||||
ASSERT_LEVELDB_OK(writable_file->Close());
|
||||
ASSERT_OK(writable_file->Sync());
|
||||
ASSERT_OK(writable_file->Flush());
|
||||
ASSERT_OK(writable_file->Close());
|
||||
delete writable_file;
|
||||
}
|
||||
|
||||
TEST_F(MemEnvTest, LargeWrite) {
|
||||
TEST(MemEnvTest, LargeWrite) {
|
||||
const size_t kWriteSize = 300 * 1024;
|
||||
char* scratch = new char[kWriteSize * 2];
|
||||
|
||||
@ -167,53 +168,53 @@ TEST_F(MemEnvTest, LargeWrite) {
|
||||
}
|
||||
|
||||
WritableFile* writable_file;
|
||||
ASSERT_LEVELDB_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_LEVELDB_OK(writable_file->Append("foo"));
|
||||
ASSERT_LEVELDB_OK(writable_file->Append(write_data));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("foo"));
|
||||
ASSERT_OK(writable_file->Append(write_data));
|
||||
delete writable_file;
|
||||
|
||||
SequentialFile* seq_file;
|
||||
Slice result;
|
||||
ASSERT_LEVELDB_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_LEVELDB_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
|
||||
ASSERT_EQ(0, result.compare("foo"));
|
||||
|
||||
size_t read = 0;
|
||||
std::string read_data;
|
||||
while (read < kWriteSize) {
|
||||
ASSERT_LEVELDB_OK(seq_file->Read(kWriteSize - read, &result, scratch));
|
||||
ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
|
||||
read_data.append(result.data(), result.size());
|
||||
read += result.size();
|
||||
}
|
||||
ASSERT_TRUE(write_data == read_data);
|
||||
delete seq_file;
|
||||
delete[] scratch;
|
||||
delete [] scratch;
|
||||
}
|
||||
|
||||
TEST_F(MemEnvTest, OverwriteOpenFile) {
|
||||
TEST(MemEnvTest, OverwriteOpenFile) {
|
||||
const char kWrite1Data[] = "Write #1 data";
|
||||
const size_t kFileDataLen = sizeof(kWrite1Data) - 1;
|
||||
const std::string kTestFileName = testing::TempDir() + "leveldb-TestFile.dat";
|
||||
const std::string kTestFileName = test::TmpDir() + "/leveldb-TestFile.dat";
|
||||
|
||||
ASSERT_LEVELDB_OK(WriteStringToFile(env_, kWrite1Data, kTestFileName));
|
||||
ASSERT_OK(WriteStringToFile(env_, kWrite1Data, kTestFileName));
|
||||
|
||||
RandomAccessFile* rand_file;
|
||||
ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(kTestFileName, &rand_file));
|
||||
ASSERT_OK(env_->NewRandomAccessFile(kTestFileName, &rand_file));
|
||||
|
||||
const char kWrite2Data[] = "Write #2 data";
|
||||
ASSERT_LEVELDB_OK(WriteStringToFile(env_, kWrite2Data, kTestFileName));
|
||||
ASSERT_OK(WriteStringToFile(env_, kWrite2Data, kTestFileName));
|
||||
|
||||
// Verify that overwriting an open file will result in the new file data
|
||||
// being read from files opened before the write.
|
||||
Slice result;
|
||||
char scratch[kFileDataLen];
|
||||
ASSERT_LEVELDB_OK(rand_file->Read(0, kFileDataLen, &result, scratch));
|
||||
ASSERT_OK(rand_file->Read(0, kFileDataLen, &result, scratch));
|
||||
ASSERT_EQ(0, result.compare(kWrite2Data));
|
||||
|
||||
delete rand_file;
|
||||
}
|
||||
|
||||
TEST_F(MemEnvTest, DBTest) {
|
||||
TEST(MemEnvTest, DBTest) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.env = env_;
|
||||
@ -222,14 +223,14 @@ TEST_F(MemEnvTest, DBTest) {
|
||||
const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
|
||||
const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
|
||||
|
||||
ASSERT_LEVELDB_OK(DB::Open(options, "/dir/db", &db));
|
||||
ASSERT_OK(DB::Open(options, "/dir/db", &db));
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
ASSERT_LEVELDB_OK(db->Put(WriteOptions(), keys[i], vals[i]));
|
||||
ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i]));
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
std::string res;
|
||||
ASSERT_LEVELDB_OK(db->Get(ReadOptions(), keys[i], &res));
|
||||
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
|
||||
ASSERT_TRUE(res == vals[i]);
|
||||
}
|
||||
|
||||
@ -245,11 +246,11 @@ TEST_F(MemEnvTest, DBTest) {
|
||||
delete iterator;
|
||||
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
|
||||
ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable());
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
std::string res;
|
||||
ASSERT_LEVELDB_OK(db->Get(ReadOptions(), keys[i], &res));
|
||||
ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
|
||||
ASSERT_TRUE(res == vals[i]);
|
||||
}
|
||||
|
||||
@ -257,3 +258,7 @@ TEST_F(MemEnvTest, DBTest) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -32,7 +32,7 @@
|
||||
On failure, leveldb frees the old value of *errptr and
|
||||
set *errptr to a malloc()ed error message.
|
||||
|
||||
(4) Bools have the type uint8_t (0 == false; rest == true)
|
||||
(4) Bools have the type unsigned char (0 == false; rest == true)
|
||||
|
||||
(5) All of the pointer arguments must be non-NULL.
|
||||
*/
|
||||
@ -40,34 +40,33 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_C_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_C_H_
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include "leveldb/export.h"
|
||||
|
||||
/* Exported types */
|
||||
|
||||
typedef struct leveldb_t leveldb_t;
|
||||
typedef struct leveldb_cache_t leveldb_cache_t;
|
||||
typedef struct leveldb_comparator_t leveldb_comparator_t;
|
||||
typedef struct leveldb_env_t leveldb_env_t;
|
||||
typedef struct leveldb_filelock_t leveldb_filelock_t;
|
||||
typedef struct leveldb_filterpolicy_t leveldb_filterpolicy_t;
|
||||
typedef struct leveldb_iterator_t leveldb_iterator_t;
|
||||
typedef struct leveldb_logger_t leveldb_logger_t;
|
||||
typedef struct leveldb_options_t leveldb_options_t;
|
||||
typedef struct leveldb_randomfile_t leveldb_randomfile_t;
|
||||
typedef struct leveldb_readoptions_t leveldb_readoptions_t;
|
||||
typedef struct leveldb_seqfile_t leveldb_seqfile_t;
|
||||
typedef struct leveldb_snapshot_t leveldb_snapshot_t;
|
||||
typedef struct leveldb_writablefile_t leveldb_writablefile_t;
|
||||
typedef struct leveldb_writebatch_t leveldb_writebatch_t;
|
||||
typedef struct leveldb_writeoptions_t leveldb_writeoptions_t;
|
||||
typedef struct leveldb_t leveldb_t;
|
||||
typedef struct leveldb_cache_t leveldb_cache_t;
|
||||
typedef struct leveldb_comparator_t leveldb_comparator_t;
|
||||
typedef struct leveldb_env_t leveldb_env_t;
|
||||
typedef struct leveldb_filelock_t leveldb_filelock_t;
|
||||
typedef struct leveldb_filterpolicy_t leveldb_filterpolicy_t;
|
||||
typedef struct leveldb_iterator_t leveldb_iterator_t;
|
||||
typedef struct leveldb_logger_t leveldb_logger_t;
|
||||
typedef struct leveldb_options_t leveldb_options_t;
|
||||
typedef struct leveldb_randomfile_t leveldb_randomfile_t;
|
||||
typedef struct leveldb_readoptions_t leveldb_readoptions_t;
|
||||
typedef struct leveldb_seqfile_t leveldb_seqfile_t;
|
||||
typedef struct leveldb_snapshot_t leveldb_snapshot_t;
|
||||
typedef struct leveldb_writablefile_t leveldb_writablefile_t;
|
||||
typedef struct leveldb_writebatch_t leveldb_writebatch_t;
|
||||
typedef struct leveldb_writeoptions_t leveldb_writeoptions_t;
|
||||
|
||||
/* DB operations */
|
||||
|
||||
@ -131,7 +130,7 @@ LEVELDB_EXPORT void leveldb_repair_db(const leveldb_options_t* options,
|
||||
/* Iterator */
|
||||
|
||||
LEVELDB_EXPORT void leveldb_iter_destroy(leveldb_iterator_t*);
|
||||
LEVELDB_EXPORT uint8_t leveldb_iter_valid(const leveldb_iterator_t*);
|
||||
LEVELDB_EXPORT unsigned char leveldb_iter_valid(const leveldb_iterator_t*);
|
||||
LEVELDB_EXPORT void leveldb_iter_seek_to_first(leveldb_iterator_t*);
|
||||
LEVELDB_EXPORT void leveldb_iter_seek_to_last(leveldb_iterator_t*);
|
||||
LEVELDB_EXPORT void leveldb_iter_seek(leveldb_iterator_t*, const char* k,
|
||||
@ -147,7 +146,7 @@ LEVELDB_EXPORT void leveldb_iter_get_error(const leveldb_iterator_t*,
|
||||
|
||||
/* Write batch */
|
||||
|
||||
LEVELDB_EXPORT leveldb_writebatch_t* leveldb_writebatch_create(void);
|
||||
LEVELDB_EXPORT leveldb_writebatch_t* leveldb_writebatch_create();
|
||||
LEVELDB_EXPORT void leveldb_writebatch_destroy(leveldb_writebatch_t*);
|
||||
LEVELDB_EXPORT void leveldb_writebatch_clear(leveldb_writebatch_t*);
|
||||
LEVELDB_EXPORT void leveldb_writebatch_put(leveldb_writebatch_t*,
|
||||
@ -164,18 +163,18 @@ LEVELDB_EXPORT void leveldb_writebatch_append(
|
||||
|
||||
/* Options */
|
||||
|
||||
LEVELDB_EXPORT leveldb_options_t* leveldb_options_create(void);
|
||||
LEVELDB_EXPORT leveldb_options_t* leveldb_options_create();
|
||||
LEVELDB_EXPORT void leveldb_options_destroy(leveldb_options_t*);
|
||||
LEVELDB_EXPORT void leveldb_options_set_comparator(leveldb_options_t*,
|
||||
leveldb_comparator_t*);
|
||||
LEVELDB_EXPORT void leveldb_options_set_filter_policy(leveldb_options_t*,
|
||||
leveldb_filterpolicy_t*);
|
||||
LEVELDB_EXPORT void leveldb_options_set_create_if_missing(leveldb_options_t*,
|
||||
uint8_t);
|
||||
unsigned char);
|
||||
LEVELDB_EXPORT void leveldb_options_set_error_if_exists(leveldb_options_t*,
|
||||
uint8_t);
|
||||
unsigned char);
|
||||
LEVELDB_EXPORT void leveldb_options_set_paranoid_checks(leveldb_options_t*,
|
||||
uint8_t);
|
||||
unsigned char);
|
||||
LEVELDB_EXPORT void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*);
|
||||
LEVELDB_EXPORT void leveldb_options_set_info_log(leveldb_options_t*,
|
||||
leveldb_logger_t*);
|
||||
@ -190,7 +189,10 @@ LEVELDB_EXPORT void leveldb_options_set_block_restart_interval(
|
||||
LEVELDB_EXPORT void leveldb_options_set_max_file_size(leveldb_options_t*,
|
||||
size_t);
|
||||
|
||||
enum { leveldb_no_compression = 0, leveldb_snappy_compression = 1 };
|
||||
enum {
|
||||
leveldb_no_compression = 0,
|
||||
leveldb_snappy_compression = 1
|
||||
};
|
||||
LEVELDB_EXPORT void leveldb_options_set_compression(leveldb_options_t*, int);
|
||||
|
||||
/* Comparator */
|
||||
@ -209,8 +211,8 @@ LEVELDB_EXPORT leveldb_filterpolicy_t* leveldb_filterpolicy_create(
|
||||
char* (*create_filter)(void*, const char* const* key_array,
|
||||
const size_t* key_length_array, int num_keys,
|
||||
size_t* filter_length),
|
||||
uint8_t (*key_may_match)(void*, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length),
|
||||
unsigned char (*key_may_match)(void*, const char* key, size_t length,
|
||||
const char* filter, size_t filter_length),
|
||||
const char* (*name)(void*));
|
||||
LEVELDB_EXPORT void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*);
|
||||
|
||||
@ -219,21 +221,21 @@ LEVELDB_EXPORT leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(
|
||||
|
||||
/* Read options */
|
||||
|
||||
LEVELDB_EXPORT leveldb_readoptions_t* leveldb_readoptions_create(void);
|
||||
LEVELDB_EXPORT leveldb_readoptions_t* leveldb_readoptions_create();
|
||||
LEVELDB_EXPORT void leveldb_readoptions_destroy(leveldb_readoptions_t*);
|
||||
LEVELDB_EXPORT void leveldb_readoptions_set_verify_checksums(
|
||||
leveldb_readoptions_t*, uint8_t);
|
||||
leveldb_readoptions_t*, unsigned char);
|
||||
LEVELDB_EXPORT void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t*,
|
||||
uint8_t);
|
||||
unsigned char);
|
||||
LEVELDB_EXPORT void leveldb_readoptions_set_snapshot(leveldb_readoptions_t*,
|
||||
const leveldb_snapshot_t*);
|
||||
|
||||
/* Write options */
|
||||
|
||||
LEVELDB_EXPORT leveldb_writeoptions_t* leveldb_writeoptions_create(void);
|
||||
LEVELDB_EXPORT leveldb_writeoptions_t* leveldb_writeoptions_create();
|
||||
LEVELDB_EXPORT void leveldb_writeoptions_destroy(leveldb_writeoptions_t*);
|
||||
LEVELDB_EXPORT void leveldb_writeoptions_set_sync(leveldb_writeoptions_t*,
|
||||
uint8_t);
|
||||
unsigned char);
|
||||
|
||||
/* Cache */
|
||||
|
||||
@ -242,7 +244,7 @@ LEVELDB_EXPORT void leveldb_cache_destroy(leveldb_cache_t* cache);
|
||||
|
||||
/* Env */
|
||||
|
||||
LEVELDB_EXPORT leveldb_env_t* leveldb_create_default_env(void);
|
||||
LEVELDB_EXPORT leveldb_env_t* leveldb_create_default_env();
|
||||
LEVELDB_EXPORT void leveldb_env_destroy(leveldb_env_t*);
|
||||
|
||||
/* If not NULL, the returned buffer must be released using leveldb_free(). */
|
||||
@ -258,13 +260,13 @@ LEVELDB_EXPORT char* leveldb_env_get_test_directory(leveldb_env_t*);
|
||||
LEVELDB_EXPORT void leveldb_free(void* ptr);
|
||||
|
||||
/* Return the major version number for this release. */
|
||||
LEVELDB_EXPORT int leveldb_major_version(void);
|
||||
LEVELDB_EXPORT int leveldb_major_version();
|
||||
|
||||
/* Return the minor version number for this release. */
|
||||
LEVELDB_EXPORT int leveldb_minor_version(void);
|
||||
LEVELDB_EXPORT int leveldb_minor_version();
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* end extern "C" */
|
||||
} /* end extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* STORAGE_LEVELDB_INCLUDE_C_H_ */
|
||||
#endif /* STORAGE_LEVELDB_INCLUDE_C_H_ */
|
||||
|
@ -18,8 +18,7 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_CACHE_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/slice.h"
|
||||
|
||||
@ -43,7 +42,7 @@ class LEVELDB_EXPORT Cache {
|
||||
virtual ~Cache();
|
||||
|
||||
// Opaque handle to an entry stored in the cache.
|
||||
struct Handle {};
|
||||
struct Handle { };
|
||||
|
||||
// Insert a mapping from key->value into the cache and assign it
|
||||
// the specified charge against the total cache capacity.
|
||||
@ -96,6 +95,14 @@ class LEVELDB_EXPORT Cache {
|
||||
// Return an estimate of the combined charges of all elements stored in the
|
||||
// cache.
|
||||
virtual size_t TotalCharge() const = 0;
|
||||
|
||||
private:
|
||||
void LRU_Remove(Handle* e);
|
||||
void LRU_Append(Handle* e);
|
||||
void Unref(Handle* e);
|
||||
|
||||
struct Rep;
|
||||
Rep* rep_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -6,7 +6,6 @@
|
||||
#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -45,8 +44,9 @@ class LEVELDB_EXPORT Comparator {
|
||||
// If *start < limit, changes *start to a short string in [start,limit).
|
||||
// Simple comparator implementations may return with *start unchanged,
|
||||
// i.e., an implementation of this method that does nothing is correct.
|
||||
virtual void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const = 0;
|
||||
virtual void FindShortestSeparator(
|
||||
std::string* start,
|
||||
const Slice& limit) const = 0;
|
||||
|
||||
// Changes *key to a short string >= *key.
|
||||
// Simple comparator implementations may return with *key unchanged,
|
||||
|
@ -5,9 +5,8 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_DB_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
@ -16,7 +15,7 @@ namespace leveldb {
|
||||
|
||||
// Update CMakeLists.txt if you change these
|
||||
static const int kMajorVersion = 1;
|
||||
static const int kMinorVersion = 23;
|
||||
static const int kMinorVersion = 21;
|
||||
|
||||
struct Options;
|
||||
struct ReadOptions;
|
||||
@ -33,11 +32,11 @@ class LEVELDB_EXPORT Snapshot {
|
||||
|
||||
// A range of keys
|
||||
struct LEVELDB_EXPORT Range {
|
||||
Range() = default;
|
||||
Range(const Slice& s, const Slice& l) : start(s), limit(l) {}
|
||||
Slice start; // Included in the range
|
||||
Slice limit; // Not included in the range
|
||||
|
||||
Slice start; // Included in the range
|
||||
Slice limit; // Not included in the range
|
||||
Range() { }
|
||||
Range(const Slice& s, const Slice& l) : start(s), limit(l) { }
|
||||
};
|
||||
|
||||
// A DB is a persistent ordered map from keys to values.
|
||||
@ -50,7 +49,8 @@ class LEVELDB_EXPORT DB {
|
||||
// OK on success.
|
||||
// Stores nullptr in *dbptr and returns a non-OK status on error.
|
||||
// Caller should delete *dbptr when it is no longer needed.
|
||||
static Status Open(const Options& options, const std::string& name,
|
||||
static Status Open(const Options& options,
|
||||
const std::string& name,
|
||||
DB** dbptr);
|
||||
|
||||
DB() = default;
|
||||
@ -63,7 +63,8 @@ class LEVELDB_EXPORT DB {
|
||||
// Set the database entry for "key" to "value". Returns OK on success,
|
||||
// and a non-OK status on error.
|
||||
// Note: consider setting options.sync = true.
|
||||
virtual Status Put(const WriteOptions& options, const Slice& key,
|
||||
virtual Status Put(const WriteOptions& options,
|
||||
const Slice& key,
|
||||
const Slice& value) = 0;
|
||||
|
||||
// Remove the database entry (if any) for "key". Returns OK on
|
||||
@ -84,8 +85,8 @@ class LEVELDB_EXPORT DB {
|
||||
// a status for which Status::IsNotFound() returns true.
|
||||
//
|
||||
// May return some other Status on an error.
|
||||
virtual Status Get(const ReadOptions& options, const Slice& key,
|
||||
std::string* value) = 0;
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key, std::string* value) = 0;
|
||||
|
||||
// Return a heap-allocated iterator over the contents of the database.
|
||||
// The result of NewIterator() is initially invalid (caller must
|
||||
|
@ -6,7 +6,6 @@
|
||||
#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/status.h"
|
||||
|
@ -13,26 +13,28 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_ENV_H_
|
||||
|
||||
#include <cstdarg>
|
||||
#include <cstdint>
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
// This workaround can be removed when leveldb::Env::DeleteFile is removed.
|
||||
#if defined(_WIN32)
|
||||
// On Windows, the method name DeleteFile (below) introduces the risk of
|
||||
// triggering undefined behavior by exposing the compiler to different
|
||||
// declarations of the Env class in different translation units.
|
||||
// The leveldb::Env class below contains a DeleteFile method.
|
||||
// At the same time, <windows.h>, a fairly popular header
|
||||
// file for Windows applications, defines a DeleteFile macro.
|
||||
//
|
||||
// This is because <windows.h>, a fairly popular header file for Windows
|
||||
// applications, defines a DeleteFile macro. So, files that include the Windows
|
||||
// header before this header will contain an altered Env declaration.
|
||||
// Without any intervention on our part, the result of this
|
||||
// unfortunate coincidence is that the name of the
|
||||
// leveldb::Env::DeleteFile method seen by the compiler depends on
|
||||
// whether <windows.h> was included before or after the LevelDB
|
||||
// headers.
|
||||
//
|
||||
// This workaround ensures that the compiler sees the same Env declaration,
|
||||
// independently of whether <windows.h> was included.
|
||||
// To avoid headaches, we undefined DeleteFile (if defined) and
|
||||
// redefine it at the bottom of this file. This way <windows.h>
|
||||
// can be included before this file (or not at all) and the
|
||||
// exported method will always be leveldb::Env::DeleteFile.
|
||||
#if defined(DeleteFile)
|
||||
#undef DeleteFile
|
||||
#define LEVELDB_DELETEFILE_UNDEFINED
|
||||
@ -50,7 +52,7 @@ class WritableFile;
|
||||
|
||||
class LEVELDB_EXPORT Env {
|
||||
public:
|
||||
Env();
|
||||
Env() = default;
|
||||
|
||||
Env(const Env&) = delete;
|
||||
Env& operator=(const Env&) = delete;
|
||||
@ -118,48 +120,15 @@ class LEVELDB_EXPORT Env {
|
||||
// Original contents of *results are dropped.
|
||||
virtual Status GetChildren(const std::string& dir,
|
||||
std::vector<std::string>* result) = 0;
|
||||
// Delete the named file.
|
||||
//
|
||||
// The default implementation calls DeleteFile, to support legacy Env
|
||||
// implementations. Updated Env implementations must override RemoveFile and
|
||||
// ignore the existence of DeleteFile. Updated code calling into the Env API
|
||||
// must call RemoveFile instead of DeleteFile.
|
||||
//
|
||||
// A future release will remove DeleteDir and the default implementation of
|
||||
// RemoveDir.
|
||||
virtual Status RemoveFile(const std::string& fname);
|
||||
|
||||
// DEPRECATED: Modern Env implementations should override RemoveFile instead.
|
||||
//
|
||||
// The default implementation calls RemoveFile, to support legacy Env user
|
||||
// code that calls this method on modern Env implementations. Modern Env user
|
||||
// code should call RemoveFile.
|
||||
//
|
||||
// A future release will remove this method.
|
||||
virtual Status DeleteFile(const std::string& fname);
|
||||
// Delete the named file.
|
||||
virtual Status DeleteFile(const std::string& fname) = 0;
|
||||
|
||||
// Create the specified directory.
|
||||
virtual Status CreateDir(const std::string& dirname) = 0;
|
||||
|
||||
// Delete the specified directory.
|
||||
//
|
||||
// The default implementation calls DeleteDir, to support legacy Env
|
||||
// implementations. Updated Env implementations must override RemoveDir and
|
||||
// ignore the existence of DeleteDir. Modern code calling into the Env API
|
||||
// must call RemoveDir instead of DeleteDir.
|
||||
//
|
||||
// A future release will remove DeleteDir and the default implementation of
|
||||
// RemoveDir.
|
||||
virtual Status RemoveDir(const std::string& dirname);
|
||||
|
||||
// DEPRECATED: Modern Env implementations should override RemoveDir instead.
|
||||
//
|
||||
// The default implementation calls RemoveDir, to support legacy Env user
|
||||
// code that calls this method on modern Env implementations. Modern Env user
|
||||
// code should call RemoveDir.
|
||||
//
|
||||
// A future release will remove this method.
|
||||
virtual Status DeleteDir(const std::string& dirname);
|
||||
virtual Status DeleteDir(const std::string& dirname) = 0;
|
||||
|
||||
// Store the size of fname in *file_size.
|
||||
virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0;
|
||||
@ -195,14 +164,16 @@ class LEVELDB_EXPORT Env {
|
||||
// added to the same Env may run concurrently in different threads.
|
||||
// I.e., the caller may not assume that background work items are
|
||||
// serialized.
|
||||
virtual void Schedule(void (*function)(void* arg), void* arg) = 0;
|
||||
virtual void Schedule(
|
||||
void (*function)(void* arg),
|
||||
void* arg) = 0;
|
||||
|
||||
// Start a new thread, invoking "function(arg)" within the new thread.
|
||||
// When "function(arg)" returns, the thread will be destroyed.
|
||||
virtual void StartThread(void (*function)(void* arg), void* arg) = 0;
|
||||
|
||||
// *path is set to a temporary directory that can be used for testing. It may
|
||||
// or may not have just been created. The directory may or may not differ
|
||||
// or many not have just been created. The directory may or may not differ
|
||||
// between runs of the same process, but subsequent calls will return the
|
||||
// same directory.
|
||||
virtual Status GetTestDirectory(std::string* path) = 0;
|
||||
@ -300,7 +271,7 @@ class LEVELDB_EXPORT Logger {
|
||||
virtual ~Logger();
|
||||
|
||||
// Write an entry to the log file with the specified format.
|
||||
virtual void Logv(const char* format, std::va_list ap) = 0;
|
||||
virtual void Logv(const char* format, va_list ap) = 0;
|
||||
};
|
||||
|
||||
// Identifies a locked file.
|
||||
@ -316,9 +287,9 @@ class LEVELDB_EXPORT FileLock {
|
||||
|
||||
// Log the specified data to *info_log if info_log is non-null.
|
||||
void Log(Logger* info_log, const char* format, ...)
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
__attribute__((__format__(__printf__, 2, 3)))
|
||||
#endif
|
||||
# if defined(__GNUC__) || defined(__clang__)
|
||||
__attribute__((__format__ (__printf__, 2, 3)))
|
||||
# endif
|
||||
;
|
||||
|
||||
// A utility routine: write "data" to the named file.
|
||||
@ -335,7 +306,7 @@ LEVELDB_EXPORT Status ReadFileToString(Env* env, const std::string& fname,
|
||||
class LEVELDB_EXPORT EnvWrapper : public Env {
|
||||
public:
|
||||
// Initialize an EnvWrapper that delegates all calls to *t.
|
||||
explicit EnvWrapper(Env* t) : target_(t) {}
|
||||
explicit EnvWrapper(Env* t) : target_(t) { }
|
||||
virtual ~EnvWrapper();
|
||||
|
||||
// Return the target to which this Env forwards all calls.
|
||||
@ -362,14 +333,14 @@ class LEVELDB_EXPORT EnvWrapper : public Env {
|
||||
std::vector<std::string>* r) override {
|
||||
return target_->GetChildren(dir, r);
|
||||
}
|
||||
Status RemoveFile(const std::string& f) override {
|
||||
return target_->RemoveFile(f);
|
||||
Status DeleteFile(const std::string& f) override {
|
||||
return target_->DeleteFile(f);
|
||||
}
|
||||
Status CreateDir(const std::string& d) override {
|
||||
return target_->CreateDir(d);
|
||||
}
|
||||
Status RemoveDir(const std::string& d) override {
|
||||
return target_->RemoveDir(d);
|
||||
Status DeleteDir(const std::string& d) override {
|
||||
return target_->DeleteDir(d);
|
||||
}
|
||||
Status GetFileSize(const std::string& f, uint64_t* s) override {
|
||||
return target_->GetFileSize(f, s);
|
||||
@ -393,7 +364,9 @@ class LEVELDB_EXPORT EnvWrapper : public Env {
|
||||
Status NewLogger(const std::string& fname, Logger** result) override {
|
||||
return target_->NewLogger(fname, result);
|
||||
}
|
||||
uint64_t NowMicros() override { return target_->NowMicros(); }
|
||||
uint64_t NowMicros() override {
|
||||
return target_->NowMicros();
|
||||
}
|
||||
void SleepForMicroseconds(int micros) override {
|
||||
target_->SleepForMicroseconds(micros);
|
||||
}
|
||||
@ -404,8 +377,7 @@ class LEVELDB_EXPORT EnvWrapper : public Env {
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
// This workaround can be removed when leveldb::Env::DeleteFile is removed.
|
||||
// Redefine DeleteFile if it was undefined earlier.
|
||||
// Redefine DeleteFile if necessary.
|
||||
#if defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED)
|
||||
#if defined(UNICODE)
|
||||
#define DeleteFile DeleteFileW
|
||||
|
@ -17,7 +17,6 @@
|
||||
#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -40,8 +39,8 @@ class LEVELDB_EXPORT FilterPolicy {
|
||||
//
|
||||
// Warning: do not change the initial contents of *dst. Instead,
|
||||
// append the newly constructed filter to *dst.
|
||||
virtual void CreateFilter(const Slice* keys, int n,
|
||||
std::string* dst) const = 0;
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst)
|
||||
const = 0;
|
||||
|
||||
// "filter" contains the data appended by a preceding call to
|
||||
// CreateFilter() on this class. This method must return true if
|
||||
|
@ -84,19 +84,16 @@ class LEVELDB_EXPORT Iterator {
|
||||
// Cleanup functions are stored in a single-linked list.
|
||||
// The list's head node is inlined in the iterator.
|
||||
struct CleanupNode {
|
||||
// True if the node is not used. Only head nodes might be unused.
|
||||
bool IsEmpty() const { return function == nullptr; }
|
||||
// Invokes the cleanup function.
|
||||
void Run() {
|
||||
assert(function != nullptr);
|
||||
(*function)(arg1, arg2);
|
||||
}
|
||||
|
||||
// The head node is used if the function pointer is not null.
|
||||
CleanupFunction function;
|
||||
void* arg1;
|
||||
void* arg2;
|
||||
CleanupNode* next;
|
||||
|
||||
// True if the node is not used. Only head nodes might be unused.
|
||||
bool IsEmpty() const { return function == nullptr; }
|
||||
// Invokes the cleanup function.
|
||||
void Run() { assert(function != nullptr); (*function)(arg1, arg2); }
|
||||
};
|
||||
CleanupNode cleanup_head_;
|
||||
};
|
||||
|
@ -5,8 +5,7 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include <stddef.h>
|
||||
#include "leveldb/export.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -25,16 +24,12 @@ class Snapshot;
|
||||
enum CompressionType {
|
||||
// NOTE: do not change the values of existing entries, as these are
|
||||
// part of the persistent format on disk.
|
||||
kNoCompression = 0x0,
|
||||
kSnappyCompression = 0x1,
|
||||
kZstdCompression = 0x2,
|
||||
kNoCompression = 0x0,
|
||||
kSnappyCompression = 0x1
|
||||
};
|
||||
|
||||
// Options to control the behavior of a database (passed to DB::Open)
|
||||
struct LEVELDB_EXPORT Options {
|
||||
// Create an Options object with default values for all fields.
|
||||
Options();
|
||||
|
||||
// -------------------
|
||||
// Parameters that affect behavior
|
||||
|
||||
@ -131,10 +126,6 @@ struct LEVELDB_EXPORT Options {
|
||||
// efficiently detect that and will switch to uncompressed mode.
|
||||
CompressionType compression = kSnappyCompression;
|
||||
|
||||
// Compression level for zstd.
|
||||
// Currently only the range [-5,22] is supported. Default is 1.
|
||||
int zstd_compression_level = 1;
|
||||
|
||||
// EXPERIMENTAL: If true, append to existing MANIFEST and log files
|
||||
// when a database is opened. This can significantly speed up open.
|
||||
//
|
||||
@ -145,6 +136,9 @@ struct LEVELDB_EXPORT Options {
|
||||
// Many applications will benefit from passing the result of
|
||||
// NewBloomFilterPolicy() here.
|
||||
const FilterPolicy* filter_policy = nullptr;
|
||||
|
||||
// Create an Options object with default values for all fields.
|
||||
Options();
|
||||
};
|
||||
|
||||
// Options that control read operations
|
||||
@ -162,12 +156,12 @@ struct LEVELDB_EXPORT ReadOptions {
|
||||
// not have been released). If "snapshot" is null, use an implicit
|
||||
// snapshot of the state at the beginning of this read operation.
|
||||
const Snapshot* snapshot = nullptr;
|
||||
|
||||
ReadOptions() = default;
|
||||
};
|
||||
|
||||
// Options that control write operations
|
||||
struct LEVELDB_EXPORT WriteOptions {
|
||||
WriteOptions() = default;
|
||||
|
||||
// If true, the write will be flushed from the operating system
|
||||
// buffer cache (by calling WritableFile::Sync()) before the write
|
||||
// is considered complete. If this flag is true, writes will be
|
||||
@ -183,6 +177,8 @@ struct LEVELDB_EXPORT WriteOptions {
|
||||
// with sync==true has similar crash semantics to a "write()"
|
||||
// system call followed by "fsync()".
|
||||
bool sync = false;
|
||||
|
||||
WriteOptions() = default;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -15,11 +15,10 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_SLICE_H_
|
||||
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
#include <assert.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -27,16 +26,16 @@ namespace leveldb {
|
||||
class LEVELDB_EXPORT Slice {
|
||||
public:
|
||||
// Create an empty slice.
|
||||
Slice() : data_(""), size_(0) {}
|
||||
Slice() : data_(""), size_(0) { }
|
||||
|
||||
// Create a slice that refers to d[0,n-1].
|
||||
Slice(const char* d, size_t n) : data_(d), size_(n) {}
|
||||
Slice(const char* d, size_t n) : data_(d), size_(n) { }
|
||||
|
||||
// Create a slice that refers to the contents of "s"
|
||||
Slice(const std::string& s) : data_(s.data()), size_(s.size()) {}
|
||||
Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
|
||||
|
||||
// Create a slice that refers to s[0,strlen(s)-1]
|
||||
Slice(const char* s) : data_(s), size_(strlen(s)) {}
|
||||
Slice(const char* s) : data_(s), size_(strlen(s)) { }
|
||||
|
||||
// Intentionally copyable.
|
||||
Slice(const Slice&) = default;
|
||||
@ -59,10 +58,7 @@ class LEVELDB_EXPORT Slice {
|
||||
}
|
||||
|
||||
// Change this slice to refer to an empty array
|
||||
void clear() {
|
||||
data_ = "";
|
||||
size_ = 0;
|
||||
}
|
||||
void clear() { data_ = ""; size_ = 0; }
|
||||
|
||||
// Drop the first "n" bytes from this slice.
|
||||
void remove_prefix(size_t n) {
|
||||
@ -82,7 +78,8 @@ class LEVELDB_EXPORT Slice {
|
||||
|
||||
// Return true iff "x" is a prefix of "*this"
|
||||
bool starts_with(const Slice& x) const {
|
||||
return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0));
|
||||
return ((size_ >= x.size_) &&
|
||||
(memcmp(data_, x.data_, x.size_) == 0));
|
||||
}
|
||||
|
||||
private:
|
||||
@ -95,20 +92,21 @@ inline bool operator==(const Slice& x, const Slice& y) {
|
||||
(memcmp(x.data(), y.data(), x.size()) == 0));
|
||||
}
|
||||
|
||||
inline bool operator!=(const Slice& x, const Slice& y) { return !(x == y); }
|
||||
inline bool operator!=(const Slice& x, const Slice& y) {
|
||||
return !(x == y);
|
||||
}
|
||||
|
||||
inline int Slice::compare(const Slice& b) const {
|
||||
const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
|
||||
int r = memcmp(data_, b.data_, min_len);
|
||||
if (r == 0) {
|
||||
if (size_ < b.size_)
|
||||
r = -1;
|
||||
else if (size_ > b.size_)
|
||||
r = +1;
|
||||
if (size_ < b.size_) r = -1;
|
||||
else if (size_ > b.size_) r = +1;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_
|
||||
|
@ -15,7 +15,6 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/slice.h"
|
||||
|
||||
@ -24,7 +23,7 @@ namespace leveldb {
|
||||
class LEVELDB_EXPORT Status {
|
||||
public:
|
||||
// Create a success status.
|
||||
Status() noexcept : state_(nullptr) {}
|
||||
Status() noexcept : state_(nullptr) { }
|
||||
~Status() { delete[] state_; }
|
||||
|
||||
Status(const Status& rhs);
|
||||
@ -76,6 +75,13 @@ class LEVELDB_EXPORT Status {
|
||||
std::string ToString() const;
|
||||
|
||||
private:
|
||||
// OK status has a null state_. Otherwise, state_ is a new[] array
|
||||
// of the following form:
|
||||
// state_[0..3] == length of message
|
||||
// state_[4] == code
|
||||
// state_[5..] == message
|
||||
const char* state_;
|
||||
|
||||
enum Code {
|
||||
kOk = 0,
|
||||
kNotFound = 1,
|
||||
@ -91,13 +97,6 @@ class LEVELDB_EXPORT Status {
|
||||
|
||||
Status(Code code, const Slice& msg, const Slice& msg2);
|
||||
static const char* CopyState(const char* s);
|
||||
|
||||
// OK status has a null state_. Otherwise, state_ is a new[] array
|
||||
// of the following form:
|
||||
// state_[0..3] == length of message
|
||||
// state_[4] == code
|
||||
// state_[5..] == message
|
||||
const char* state_;
|
||||
};
|
||||
|
||||
inline Status::Status(const Status& rhs) {
|
||||
|
@ -5,8 +5,7 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_TABLE_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/iterator.h"
|
||||
|
||||
@ -37,11 +36,13 @@ class LEVELDB_EXPORT Table {
|
||||
// for the duration of the returned table's lifetime.
|
||||
//
|
||||
// *file must remain live while this Table is in use.
|
||||
static Status Open(const Options& options, RandomAccessFile* file,
|
||||
uint64_t file_size, Table** table);
|
||||
static Status Open(const Options& options,
|
||||
RandomAccessFile* file,
|
||||
uint64_t file_size,
|
||||
Table** table);
|
||||
|
||||
Table(const Table&) = delete;
|
||||
Table& operator=(const Table&) = delete;
|
||||
void operator=(const Table&) = delete;
|
||||
|
||||
~Table();
|
||||
|
||||
@ -59,24 +60,24 @@ class LEVELDB_EXPORT Table {
|
||||
uint64_t ApproximateOffsetOf(const Slice& key) const;
|
||||
|
||||
private:
|
||||
friend class TableCache;
|
||||
struct Rep;
|
||||
Rep* rep_;
|
||||
|
||||
explicit Table(Rep* rep) { rep_ = rep; }
|
||||
static Iterator* BlockReader(void*, const ReadOptions&, const Slice&);
|
||||
|
||||
explicit Table(Rep* rep) : rep_(rep) {}
|
||||
|
||||
// Calls (*handle_result)(arg, ...) with the entry found after a call
|
||||
// to Seek(key). May not make such a call if filter policy says
|
||||
// that key is not present.
|
||||
Status InternalGet(const ReadOptions&, const Slice& key, void* arg,
|
||||
void (*handle_result)(void* arg, const Slice& k,
|
||||
const Slice& v));
|
||||
friend class TableCache;
|
||||
Status InternalGet(
|
||||
const ReadOptions&, const Slice& key,
|
||||
void* arg,
|
||||
void (*handle_result)(void* arg, const Slice& k, const Slice& v));
|
||||
|
||||
|
||||
void ReadMeta(const Footer& footer);
|
||||
void ReadFilter(const Slice& filter_handle_value);
|
||||
|
||||
Rep* const rep_;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -13,8 +13,7 @@
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/status.h"
|
||||
@ -33,7 +32,7 @@ class LEVELDB_EXPORT TableBuilder {
|
||||
TableBuilder(const Options& options, WritableFile* file);
|
||||
|
||||
TableBuilder(const TableBuilder&) = delete;
|
||||
TableBuilder& operator=(const TableBuilder&) = delete;
|
||||
void operator=(const TableBuilder&) = delete;
|
||||
|
||||
// REQUIRES: Either Finish() or Abandon() has been called.
|
||||
~TableBuilder();
|
||||
|
@ -22,7 +22,6 @@
|
||||
#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "leveldb/export.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
@ -32,18 +31,11 @@ class Slice;
|
||||
|
||||
class LEVELDB_EXPORT WriteBatch {
|
||||
public:
|
||||
class LEVELDB_EXPORT Handler {
|
||||
public:
|
||||
virtual ~Handler();
|
||||
virtual void Put(const Slice& key, const Slice& value) = 0;
|
||||
virtual void Delete(const Slice& key) = 0;
|
||||
};
|
||||
|
||||
WriteBatch();
|
||||
|
||||
// Intentionally copyable.
|
||||
WriteBatch(const WriteBatch&) = default;
|
||||
WriteBatch& operator=(const WriteBatch&) = default;
|
||||
WriteBatch& operator =(const WriteBatch&) = default;
|
||||
|
||||
~WriteBatch();
|
||||
|
||||
@ -70,6 +62,12 @@ class LEVELDB_EXPORT WriteBatch {
|
||||
void Append(const WriteBatch& source);
|
||||
|
||||
// Support for iterating over the contents of a batch.
|
||||
class Handler {
|
||||
public:
|
||||
virtual ~Handler();
|
||||
virtual void Put(const Slice& key, const Slice& value) = 0;
|
||||
virtual void Delete(const Slice& key) = 0;
|
||||
};
|
||||
Status Iterate(Handler* handler) const;
|
||||
|
||||
private:
|
||||
|
@ -3,14 +3,13 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
// Test for issue 178: a manual compaction causes deleted data to reappear.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/testutil.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@ -18,15 +17,19 @@ const int kNumKeys = 1100000;
|
||||
|
||||
std::string Key1(int i) {
|
||||
char buf[100];
|
||||
std::snprintf(buf, sizeof(buf), "my_key_%d", i);
|
||||
snprintf(buf, sizeof(buf), "my_key_%d", i);
|
||||
return buf;
|
||||
}
|
||||
|
||||
std::string Key2(int i) { return Key1(i) + "_xxx"; }
|
||||
std::string Key2(int i) {
|
||||
return Key1(i) + "_xxx";
|
||||
}
|
||||
|
||||
class Issue178 { };
|
||||
|
||||
TEST(Issue178, Test) {
|
||||
// Get rid of any state from an old run.
|
||||
std::string dbpath = testing::TempDir() + "leveldb_cbug_test";
|
||||
std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test";
|
||||
DestroyDB(dbpath, leveldb::Options());
|
||||
|
||||
// Open database. Disable compression since it affects the creation
|
||||
@ -36,28 +39,28 @@ TEST(Issue178, Test) {
|
||||
leveldb::Options db_options;
|
||||
db_options.create_if_missing = true;
|
||||
db_options.compression = leveldb::kNoCompression;
|
||||
ASSERT_LEVELDB_OK(leveldb::DB::Open(db_options, dbpath, &db));
|
||||
ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db));
|
||||
|
||||
// create first key range
|
||||
leveldb::WriteBatch batch;
|
||||
for (size_t i = 0; i < kNumKeys; i++) {
|
||||
batch.Put(Key1(i), "value for range 1 key");
|
||||
}
|
||||
ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch));
|
||||
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
|
||||
|
||||
// create second key range
|
||||
batch.Clear();
|
||||
for (size_t i = 0; i < kNumKeys; i++) {
|
||||
batch.Put(Key2(i), "value for range 2 key");
|
||||
}
|
||||
ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch));
|
||||
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
|
||||
|
||||
// delete second key range
|
||||
batch.Clear();
|
||||
for (size_t i = 0; i < kNumKeys; i++) {
|
||||
batch.Delete(Key2(i));
|
||||
}
|
||||
ASSERT_LEVELDB_OK(db->Write(leveldb::WriteOptions(), &batch));
|
||||
ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
|
||||
|
||||
// compact database
|
||||
std::string start_key = Key1(0);
|
||||
@ -83,3 +86,7 @@ TEST(Issue178, Test) {
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -6,34 +6,35 @@
|
||||
// to forward, the current key can be yielded unexpectedly if a new
|
||||
// mutation has been added just before the current key.
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "util/testutil.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Issue200 { };
|
||||
|
||||
TEST(Issue200, Test) {
|
||||
// Get rid of any state from an old run.
|
||||
std::string dbpath = testing::TempDir() + "leveldb_issue200_test";
|
||||
std::string dbpath = test::TmpDir() + "/leveldb_issue200_test";
|
||||
DestroyDB(dbpath, Options());
|
||||
|
||||
DB* db;
|
||||
DB *db;
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db));
|
||||
ASSERT_OK(DB::Open(options, dbpath, &db));
|
||||
|
||||
WriteOptions write_options;
|
||||
ASSERT_LEVELDB_OK(db->Put(write_options, "1", "b"));
|
||||
ASSERT_LEVELDB_OK(db->Put(write_options, "2", "c"));
|
||||
ASSERT_LEVELDB_OK(db->Put(write_options, "3", "d"));
|
||||
ASSERT_LEVELDB_OK(db->Put(write_options, "4", "e"));
|
||||
ASSERT_LEVELDB_OK(db->Put(write_options, "5", "f"));
|
||||
ASSERT_OK(db->Put(write_options, "1", "b"));
|
||||
ASSERT_OK(db->Put(write_options, "2", "c"));
|
||||
ASSERT_OK(db->Put(write_options, "3", "d"));
|
||||
ASSERT_OK(db->Put(write_options, "4", "e"));
|
||||
ASSERT_OK(db->Put(write_options, "5", "f"));
|
||||
|
||||
ReadOptions read_options;
|
||||
Iterator* iter = db->NewIterator(read_options);
|
||||
Iterator *iter = db->NewIterator(read_options);
|
||||
|
||||
// Add an element that should not be reflected in the iterator.
|
||||
ASSERT_LEVELDB_OK(db->Put(write_options, "25", "cd"));
|
||||
ASSERT_OK(db->Put(write_options, "25", "cd"));
|
||||
|
||||
iter->Seek("5");
|
||||
ASSERT_EQ(iter->key().ToString(), "5");
|
||||
@ -52,3 +53,7 @@ TEST(Issue200, Test) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -9,10 +9,9 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/testutil.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
@ -38,6 +37,8 @@ std::string CreateRandomString(int32_t index) {
|
||||
|
||||
} // namespace
|
||||
|
||||
class Issue320 {};
|
||||
|
||||
TEST(Issue320, Test) {
|
||||
std::srand(0);
|
||||
|
||||
@ -52,8 +53,8 @@ TEST(Issue320, Test) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
|
||||
std::string dbpath = testing::TempDir() + "leveldb_issue320_test";
|
||||
ASSERT_LEVELDB_OK(DB::Open(options, dbpath, &db));
|
||||
std::string dbpath = test::TmpDir() + "/leveldb_issue320_test";
|
||||
ASSERT_OK(DB::Open(options, dbpath, &db));
|
||||
|
||||
uint32_t target_size = 10000;
|
||||
uint32_t num_items = 0;
|
||||
@ -77,8 +78,7 @@ TEST(Issue320, Test) {
|
||||
CreateRandomString(index), CreateRandomString(index)));
|
||||
batch.Put(test_map[index]->first, test_map[index]->second);
|
||||
} else {
|
||||
ASSERT_LEVELDB_OK(
|
||||
db->Get(readOptions, test_map[index]->first, &old_value));
|
||||
ASSERT_OK(db->Get(readOptions, test_map[index]->first, &old_value));
|
||||
if (old_value != test_map[index]->second) {
|
||||
std::cout << "ERROR incorrect value returned by Get" << std::endl;
|
||||
std::cout << " count=" << count << std::endl;
|
||||
@ -102,7 +102,7 @@ TEST(Issue320, Test) {
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_LEVELDB_OK(db->Write(writeOptions, &batch));
|
||||
ASSERT_OK(db->Write(writeOptions, &batch));
|
||||
|
||||
if (keep_snapshots && GenerateRandomNumber(10) == 0) {
|
||||
int i = GenerateRandomNumber(snapshots.size());
|
||||
@ -124,3 +124,5 @@ TEST(Issue320, Test) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
|
||||
|
@ -11,9 +11,9 @@
|
||||
// porting to a new platform, see "port_example.h" for documentation
|
||||
// of what the new port_<platform>.h file must provide.
|
||||
#if defined(LEVELDB_PLATFORM_POSIX) || defined(LEVELDB_PLATFORM_WINDOWS)
|
||||
#include "port/port_stdcxx.h"
|
||||
# include "port/port_stdcxx.h"
|
||||
#elif defined(LEVELDB_PLATFORM_CHROMIUM)
|
||||
#include "port/port_chromium.h"
|
||||
# include "port/port_chromium.h"
|
||||
#endif
|
||||
|
||||
#endif // STORAGE_LEVELDB_PORT_PORT_H_
|
||||
|
@ -15,11 +15,6 @@
|
||||
#cmakedefine01 HAVE_FULLFSYNC
|
||||
#endif // !defined(HAVE_FULLFSYNC)
|
||||
|
||||
// Define to 1 if you have a definition for O_CLOEXEC in <fcntl.h>.
|
||||
#if !defined(HAVE_O_CLOEXEC)
|
||||
#cmakedefine01 HAVE_O_CLOEXEC
|
||||
#endif // !defined(HAVE_O_CLOEXEC)
|
||||
|
||||
// Define to 1 if you have Google CRC32C.
|
||||
#if !defined(HAVE_CRC32C)
|
||||
#cmakedefine01 HAVE_CRC32C
|
||||
@ -30,9 +25,10 @@
|
||||
#cmakedefine01 HAVE_SNAPPY
|
||||
#endif // !defined(HAVE_SNAPPY)
|
||||
|
||||
// Define to 1 if you have Zstd.
|
||||
#if !defined(HAVE_Zstd)
|
||||
#cmakedefine01 HAVE_ZSTD
|
||||
#endif // !defined(HAVE_ZSTD)
|
||||
// Define to 1 if your processor stores words with the most significant byte
|
||||
// first (like Motorola and SPARC, unlike Intel and VAX).
|
||||
#if !defined(LEVELDB_IS_BIG_ENDIAN)
|
||||
#cmakedefine01 LEVELDB_IS_BIG_ENDIAN
|
||||
#endif // !defined(LEVELDB_IS_BIG_ENDIAN)
|
||||
|
||||
#endif // STORAGE_LEVELDB_PORT_PORT_CONFIG_H_
|
||||
#endif // STORAGE_LEVELDB_PORT_PORT_CONFIG_H_
|
@ -18,6 +18,10 @@ namespace port {
|
||||
// TODO(jorlow): Many of these belong more in the environment class rather than
|
||||
// here. We should try moving them and see if it affects perf.
|
||||
|
||||
// The following boolean constant must be true on a little-endian machine
|
||||
// and false otherwise.
|
||||
static const bool kLittleEndian = true /* or some other expression */;
|
||||
|
||||
// ------------------ Threading -------------------
|
||||
|
||||
// A Mutex represents an exclusive lock.
|
||||
@ -55,7 +59,7 @@ class CondVar {
|
||||
void Signal();
|
||||
|
||||
// Wake up all waiting threads.
|
||||
void SignalAll();
|
||||
void SignallAll();
|
||||
};
|
||||
|
||||
// ------------------ Compression -------------------
|
||||
@ -72,7 +76,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length,
|
||||
size_t* result);
|
||||
|
||||
// Attempt to snappy uncompress input[0,input_length-1] into *output.
|
||||
// Returns true if successful, false if the input is invalid snappy
|
||||
// Returns true if successful, false if the input is invalid lightweight
|
||||
// compressed data.
|
||||
//
|
||||
// REQUIRES: at least the first "n" bytes of output[] must be writable
|
||||
@ -81,26 +85,6 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length,
|
||||
bool Snappy_Uncompress(const char* input_data, size_t input_length,
|
||||
char* output);
|
||||
|
||||
// Store the zstd compression of "input[0,input_length-1]" in *output.
|
||||
// Returns false if zstd is not supported by this port.
|
||||
bool Zstd_Compress(int level, const char* input, size_t input_length,
|
||||
std::string* output);
|
||||
|
||||
// If input[0,input_length-1] looks like a valid zstd compressed
|
||||
// buffer, store the size of the uncompressed data in *result and
|
||||
// return true. Else return false.
|
||||
bool Zstd_GetUncompressedLength(const char* input, size_t length,
|
||||
size_t* result);
|
||||
|
||||
// Attempt to zstd uncompress input[0,input_length-1] into *output.
|
||||
// Returns true if successful, false if the input is invalid zstd
|
||||
// compressed data.
|
||||
//
|
||||
// REQUIRES: at least the first "n" bytes of output[] must be writable
|
||||
// where "n" is the result of a successful call to
|
||||
// Zstd_GetUncompressedLength.
|
||||
bool Zstd_Uncompress(const char* input_data, size_t input_length, char* output);
|
||||
|
||||
// ------------------ Miscellaneous -------------------
|
||||
|
||||
// If heap profiling is not supported, returns false.
|
||||
|
@ -28,16 +28,12 @@
|
||||
#if HAVE_SNAPPY
|
||||
#include <snappy.h>
|
||||
#endif // HAVE_SNAPPY
|
||||
#if HAVE_ZSTD
|
||||
#define ZSTD_STATIC_LINKING_ONLY // For ZSTD_compressionParameters.
|
||||
#include <zstd.h>
|
||||
#endif // HAVE_ZSTD
|
||||
|
||||
#include <cassert>
|
||||
#include <condition_variable> // NOLINT
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <mutex> // NOLINT
|
||||
#include <condition_variable> // NOLINT
|
||||
#include <mutex> // NOLINT
|
||||
#include <string>
|
||||
|
||||
#include "port/thread_annotations.h"
|
||||
@ -45,6 +41,8 @@
|
||||
namespace leveldb {
|
||||
namespace port {
|
||||
|
||||
static const bool kLittleEndian = !LEVELDB_IS_BIG_ENDIAN;
|
||||
|
||||
class CondVar;
|
||||
|
||||
// Thinly wraps std::mutex.
|
||||
@ -58,7 +56,7 @@ class LOCKABLE Mutex {
|
||||
|
||||
void Lock() EXCLUSIVE_LOCK_FUNCTION() { mu_.lock(); }
|
||||
void Unlock() UNLOCK_FUNCTION() { mu_.unlock(); }
|
||||
void AssertHeld() ASSERT_EXCLUSIVE_LOCK() {}
|
||||
void AssertHeld() ASSERT_EXCLUSIVE_LOCK() { }
|
||||
|
||||
private:
|
||||
friend class CondVar;
|
||||
@ -81,7 +79,6 @@ class CondVar {
|
||||
}
|
||||
void Signal() { cv_.notify_one(); }
|
||||
void SignalAll() { cv_.notify_all(); }
|
||||
|
||||
private:
|
||||
std::condition_variable cv_;
|
||||
Mutex* const mu_;
|
||||
@ -97,9 +94,7 @@ inline bool Snappy_Compress(const char* input, size_t length,
|
||||
return true;
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)input;
|
||||
(void)length;
|
||||
(void)output;
|
||||
(void)input; (void)length; (void)output;
|
||||
#endif // HAVE_SNAPPY
|
||||
|
||||
return false;
|
||||
@ -111,9 +106,7 @@ inline bool Snappy_GetUncompressedLength(const char* input, size_t length,
|
||||
return snappy::GetUncompressedLength(input, length, result);
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)input;
|
||||
(void)length;
|
||||
(void)result;
|
||||
(void)input; (void)length; (void)result;
|
||||
return false;
|
||||
#endif // HAVE_SNAPPY
|
||||
}
|
||||
@ -123,85 +116,14 @@ inline bool Snappy_Uncompress(const char* input, size_t length, char* output) {
|
||||
return snappy::RawUncompress(input, length, output);
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)input;
|
||||
(void)length;
|
||||
(void)output;
|
||||
(void)input; (void)length; (void)output;
|
||||
return false;
|
||||
#endif // HAVE_SNAPPY
|
||||
}
|
||||
|
||||
inline bool Zstd_Compress(int level, const char* input, size_t length,
|
||||
std::string* output) {
|
||||
#if HAVE_ZSTD
|
||||
// Get the MaxCompressedLength.
|
||||
size_t outlen = ZSTD_compressBound(length);
|
||||
if (ZSTD_isError(outlen)) {
|
||||
return false;
|
||||
}
|
||||
output->resize(outlen);
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_compressionParameters parameters =
|
||||
ZSTD_getCParams(level, std::max(length, size_t{1}), /*dictSize=*/0);
|
||||
ZSTD_CCtx_setCParams(ctx, parameters);
|
||||
outlen = ZSTD_compress2(ctx, &(*output)[0], output->size(), input, length);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
if (ZSTD_isError(outlen)) {
|
||||
return false;
|
||||
}
|
||||
output->resize(outlen);
|
||||
return true;
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)level;
|
||||
(void)input;
|
||||
(void)length;
|
||||
(void)output;
|
||||
return false;
|
||||
#endif // HAVE_ZSTD
|
||||
}
|
||||
|
||||
inline bool Zstd_GetUncompressedLength(const char* input, size_t length,
|
||||
size_t* result) {
|
||||
#if HAVE_ZSTD
|
||||
size_t size = ZSTD_getFrameContentSize(input, length);
|
||||
if (size == 0) return false;
|
||||
*result = size;
|
||||
return true;
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)input;
|
||||
(void)length;
|
||||
(void)result;
|
||||
return false;
|
||||
#endif // HAVE_ZSTD
|
||||
}
|
||||
|
||||
inline bool Zstd_Uncompress(const char* input, size_t length, char* output) {
|
||||
#if HAVE_ZSTD
|
||||
size_t outlen;
|
||||
if (!Zstd_GetUncompressedLength(input, length, &outlen)) {
|
||||
return false;
|
||||
}
|
||||
ZSTD_DCtx* ctx = ZSTD_createDCtx();
|
||||
outlen = ZSTD_decompressDCtx(ctx, output, outlen, input, length);
|
||||
ZSTD_freeDCtx(ctx);
|
||||
if (ZSTD_isError(outlen)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)input;
|
||||
(void)length;
|
||||
(void)output;
|
||||
return false;
|
||||
#endif // HAVE_ZSTD
|
||||
}
|
||||
|
||||
inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) {
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)func;
|
||||
(void)arg;
|
||||
(void)func; (void)arg;
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -210,9 +132,7 @@ inline uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) {
|
||||
return ::crc32c::Extend(crc, reinterpret_cast<const uint8_t*>(buf), size);
|
||||
#else
|
||||
// Silence compiler warnings about unused arguments.
|
||||
(void)crc;
|
||||
(void)buf;
|
||||
(void)size;
|
||||
(void)crc; (void)buf; (void)size;
|
||||
return 0;
|
||||
#endif // HAVE_CRC32C
|
||||
}
|
||||
|
@ -13,9 +13,9 @@
|
||||
|
||||
#if defined(__clang__)
|
||||
|
||||
#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
|
||||
#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
|
||||
#else
|
||||
#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op
|
||||
#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op
|
||||
#endif
|
||||
|
||||
#endif // !defined(THREAD_ANNOTATION_ATTRIBUTE__)
|
||||
@ -54,15 +54,18 @@
|
||||
#endif
|
||||
|
||||
#ifndef LOCK_RETURNED
|
||||
#define LOCK_RETURNED(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
|
||||
#define LOCK_RETURNED(x) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
|
||||
#endif
|
||||
|
||||
#ifndef LOCKABLE
|
||||
#define LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(lockable)
|
||||
#define LOCKABLE \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(lockable)
|
||||
#endif
|
||||
|
||||
#ifndef SCOPED_LOCKABLE
|
||||
#define SCOPED_LOCKABLE THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
|
||||
#define SCOPED_LOCKABLE \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
|
||||
#endif
|
||||
|
||||
#ifndef EXCLUSIVE_LOCK_FUNCTION
|
||||
|
@ -6,10 +6,8 @@
|
||||
|
||||
#include "table/block.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include <algorithm>
|
||||
#include "leveldb/comparator.h"
|
||||
#include "table/format.h"
|
||||
#include "util/coding.h"
|
||||
@ -29,7 +27,7 @@ Block::Block(const BlockContents& contents)
|
||||
if (size_ < sizeof(uint32_t)) {
|
||||
size_ = 0; // Error marker
|
||||
} else {
|
||||
size_t max_restarts_allowed = (size_ - sizeof(uint32_t)) / sizeof(uint32_t);
|
||||
size_t max_restarts_allowed = (size_-sizeof(uint32_t)) / sizeof(uint32_t);
|
||||
if (NumRestarts() > max_restarts_allowed) {
|
||||
// The size is too small for NumRestarts()
|
||||
size_ = 0;
|
||||
@ -53,12 +51,13 @@ Block::~Block() {
|
||||
// If any errors are detected, returns nullptr. Otherwise, returns a
|
||||
// pointer to the key delta (just past the three decoded values).
|
||||
static inline const char* DecodeEntry(const char* p, const char* limit,
|
||||
uint32_t* shared, uint32_t* non_shared,
|
||||
uint32_t* shared,
|
||||
uint32_t* non_shared,
|
||||
uint32_t* value_length) {
|
||||
if (limit - p < 3) return nullptr;
|
||||
*shared = reinterpret_cast<const uint8_t*>(p)[0];
|
||||
*non_shared = reinterpret_cast<const uint8_t*>(p)[1];
|
||||
*value_length = reinterpret_cast<const uint8_t*>(p)[2];
|
||||
*shared = reinterpret_cast<const unsigned char*>(p)[0];
|
||||
*non_shared = reinterpret_cast<const unsigned char*>(p)[1];
|
||||
*value_length = reinterpret_cast<const unsigned char*>(p)[2];
|
||||
if ((*shared | *non_shared | *value_length) < 128) {
|
||||
// Fast path: all three values are encoded in one byte each
|
||||
p += 3;
|
||||
@ -77,9 +76,9 @@ static inline const char* DecodeEntry(const char* p, const char* limit,
|
||||
class Block::Iter : public Iterator {
|
||||
private:
|
||||
const Comparator* const comparator_;
|
||||
const char* const data_; // underlying block contents
|
||||
uint32_t const restarts_; // Offset of restart array (list of fixed32)
|
||||
uint32_t const num_restarts_; // Number of uint32_t entries in restart array
|
||||
const char* const data_; // underlying block contents
|
||||
uint32_t const restarts_; // Offset of restart array (list of fixed32)
|
||||
uint32_t const num_restarts_; // Number of uint32_t entries in restart array
|
||||
|
||||
// current_ is offset in data_ of current entry. >= restarts_ if !Valid
|
||||
uint32_t current_;
|
||||
@ -113,7 +112,9 @@ class Block::Iter : public Iterator {
|
||||
}
|
||||
|
||||
public:
|
||||
Iter(const Comparator* comparator, const char* data, uint32_t restarts,
|
||||
Iter(const Comparator* comparator,
|
||||
const char* data,
|
||||
uint32_t restarts,
|
||||
uint32_t num_restarts)
|
||||
: comparator_(comparator),
|
||||
data_(data),
|
||||
@ -124,23 +125,23 @@ class Block::Iter : public Iterator {
|
||||
assert(num_restarts_ > 0);
|
||||
}
|
||||
|
||||
bool Valid() const override { return current_ < restarts_; }
|
||||
Status status() const override { return status_; }
|
||||
Slice key() const override {
|
||||
virtual bool Valid() const { return current_ < restarts_; }
|
||||
virtual Status status() const { return status_; }
|
||||
virtual Slice key() const {
|
||||
assert(Valid());
|
||||
return key_;
|
||||
}
|
||||
Slice value() const override {
|
||||
virtual Slice value() const {
|
||||
assert(Valid());
|
||||
return value_;
|
||||
}
|
||||
|
||||
void Next() override {
|
||||
virtual void Next() {
|
||||
assert(Valid());
|
||||
ParseNextKey();
|
||||
}
|
||||
|
||||
void Prev() override {
|
||||
virtual void Prev() {
|
||||
assert(Valid());
|
||||
|
||||
// Scan backwards to a restart point before current_
|
||||
@ -161,36 +162,18 @@ class Block::Iter : public Iterator {
|
||||
} while (ParseNextKey() && NextEntryOffset() < original);
|
||||
}
|
||||
|
||||
void Seek(const Slice& target) override {
|
||||
virtual void Seek(const Slice& target) {
|
||||
// Binary search in restart array to find the last restart point
|
||||
// with a key < target
|
||||
uint32_t left = 0;
|
||||
uint32_t right = num_restarts_ - 1;
|
||||
int current_key_compare = 0;
|
||||
|
||||
if (Valid()) {
|
||||
// If we're already scanning, use the current position as a starting
|
||||
// point. This is beneficial if the key we're seeking to is ahead of the
|
||||
// current position.
|
||||
current_key_compare = Compare(key_, target);
|
||||
if (current_key_compare < 0) {
|
||||
// key_ is smaller than target
|
||||
left = restart_index_;
|
||||
} else if (current_key_compare > 0) {
|
||||
right = restart_index_;
|
||||
} else {
|
||||
// We're seeking to the key we're already at.
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
while (left < right) {
|
||||
uint32_t mid = (left + right + 1) / 2;
|
||||
uint32_t region_offset = GetRestartPoint(mid);
|
||||
uint32_t shared, non_shared, value_length;
|
||||
const char* key_ptr =
|
||||
DecodeEntry(data_ + region_offset, data_ + restarts_, &shared,
|
||||
&non_shared, &value_length);
|
||||
const char* key_ptr = DecodeEntry(data_ + region_offset,
|
||||
data_ + restarts_,
|
||||
&shared, &non_shared, &value_length);
|
||||
if (key_ptr == nullptr || (shared != 0)) {
|
||||
CorruptionError();
|
||||
return;
|
||||
@ -207,15 +190,8 @@ class Block::Iter : public Iterator {
|
||||
}
|
||||
}
|
||||
|
||||
// We might be able to use our current position within the restart block.
|
||||
// This is true if we determined the key we desire is in the current block
|
||||
// and is after than the current key.
|
||||
assert(current_key_compare == 0 || Valid());
|
||||
bool skip_seek = left == restart_index_ && current_key_compare < 0;
|
||||
if (!skip_seek) {
|
||||
SeekToRestartPoint(left);
|
||||
}
|
||||
// Linear search (within restart block) for first key >= target
|
||||
SeekToRestartPoint(left);
|
||||
while (true) {
|
||||
if (!ParseNextKey()) {
|
||||
return;
|
||||
@ -226,12 +202,12 @@ class Block::Iter : public Iterator {
|
||||
}
|
||||
}
|
||||
|
||||
void SeekToFirst() override {
|
||||
virtual void SeekToFirst() {
|
||||
SeekToRestartPoint(0);
|
||||
ParseNextKey();
|
||||
}
|
||||
|
||||
void SeekToLast() override {
|
||||
virtual void SeekToLast() {
|
||||
SeekToRestartPoint(num_restarts_ - 1);
|
||||
while (ParseNextKey() && NextEntryOffset() < restarts_) {
|
||||
// Keep skipping
|
||||
@ -277,7 +253,7 @@ class Block::Iter : public Iterator {
|
||||
}
|
||||
};
|
||||
|
||||
Iterator* Block::NewIterator(const Comparator* comparator) {
|
||||
Iterator* Block::NewIterator(const Comparator* cmp) {
|
||||
if (size_ < sizeof(uint32_t)) {
|
||||
return NewErrorIterator(Status::Corruption("bad block contents"));
|
||||
}
|
||||
@ -285,7 +261,7 @@ Iterator* Block::NewIterator(const Comparator* comparator) {
|
||||
if (num_restarts == 0) {
|
||||
return NewEmptyIterator();
|
||||
} else {
|
||||
return new Iter(comparator, data_, restart_offset_, num_restarts);
|
||||
return new Iter(cmp, data_, restart_offset_, num_restarts);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -5,9 +5,8 @@
|
||||
#ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_
|
||||
#define STORAGE_LEVELDB_TABLE_BLOCK_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include "leveldb/iterator.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -20,23 +19,24 @@ class Block {
|
||||
// Initialize the block with the specified contents.
|
||||
explicit Block(const BlockContents& contents);
|
||||
|
||||
Block(const Block&) = delete;
|
||||
Block& operator=(const Block&) = delete;
|
||||
|
||||
~Block();
|
||||
|
||||
size_t size() const { return size_; }
|
||||
Iterator* NewIterator(const Comparator* comparator);
|
||||
|
||||
private:
|
||||
class Iter;
|
||||
|
||||
uint32_t NumRestarts() const;
|
||||
|
||||
const char* data_;
|
||||
size_t size_;
|
||||
uint32_t restart_offset_; // Offset in data_ of restart array
|
||||
bool owned_; // Block owns data_[]
|
||||
uint32_t restart_offset_; // Offset in data_ of restart array
|
||||
bool owned_; // Block owns data_[]
|
||||
|
||||
// No copying allowed
|
||||
Block(const Block&);
|
||||
void operator=(const Block&);
|
||||
|
||||
class Iter;
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -29,33 +29,35 @@
|
||||
#include "table/block_builder.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
|
||||
#include <assert.h>
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/table_builder.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
BlockBuilder::BlockBuilder(const Options* options)
|
||||
: options_(options), restarts_(), counter_(0), finished_(false) {
|
||||
: options_(options),
|
||||
restarts_(),
|
||||
counter_(0),
|
||||
finished_(false) {
|
||||
assert(options->block_restart_interval >= 1);
|
||||
restarts_.push_back(0); // First restart point is at offset 0
|
||||
restarts_.push_back(0); // First restart point is at offset 0
|
||||
}
|
||||
|
||||
void BlockBuilder::Reset() {
|
||||
buffer_.clear();
|
||||
restarts_.clear();
|
||||
restarts_.push_back(0); // First restart point is at offset 0
|
||||
restarts_.push_back(0); // First restart point is at offset 0
|
||||
counter_ = 0;
|
||||
finished_ = false;
|
||||
last_key_.clear();
|
||||
}
|
||||
|
||||
size_t BlockBuilder::CurrentSizeEstimate() const {
|
||||
return (buffer_.size() + // Raw data buffer
|
||||
restarts_.size() * sizeof(uint32_t) + // Restart array
|
||||
sizeof(uint32_t)); // Restart array length
|
||||
return (buffer_.size() + // Raw data buffer
|
||||
restarts_.size() * sizeof(uint32_t) + // Restart array
|
||||
sizeof(uint32_t)); // Restart array length
|
||||
}
|
||||
|
||||
Slice BlockBuilder::Finish() {
|
||||
@ -72,7 +74,7 @@ void BlockBuilder::Add(const Slice& key, const Slice& value) {
|
||||
Slice last_key_piece(last_key_);
|
||||
assert(!finished_);
|
||||
assert(counter_ <= options_->block_restart_interval);
|
||||
assert(buffer_.empty() // No values yet?
|
||||
assert(buffer_.empty() // No values yet?
|
||||
|| options_->comparator->Compare(key, last_key_piece) > 0);
|
||||
size_t shared = 0;
|
||||
if (counter_ < options_->block_restart_interval) {
|
||||
|
@ -5,9 +5,9 @@
|
||||
#ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
|
||||
#define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/slice.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -18,9 +18,6 @@ class BlockBuilder {
|
||||
public:
|
||||
explicit BlockBuilder(const Options* options);
|
||||
|
||||
BlockBuilder(const BlockBuilder&) = delete;
|
||||
BlockBuilder& operator=(const BlockBuilder&) = delete;
|
||||
|
||||
// Reset the contents as if the BlockBuilder was just constructed.
|
||||
void Reset();
|
||||
|
||||
@ -38,15 +35,21 @@ class BlockBuilder {
|
||||
size_t CurrentSizeEstimate() const;
|
||||
|
||||
// Return true iff no entries have been added since the last Reset()
|
||||
bool empty() const { return buffer_.empty(); }
|
||||
bool empty() const {
|
||||
return buffer_.empty();
|
||||
}
|
||||
|
||||
private:
|
||||
const Options* options_;
|
||||
std::string buffer_; // Destination buffer
|
||||
std::vector<uint32_t> restarts_; // Restart points
|
||||
int counter_; // Number of entries emitted since restart
|
||||
bool finished_; // Has Finish() been called?
|
||||
std::string last_key_;
|
||||
const Options* options_;
|
||||
std::string buffer_; // Destination buffer
|
||||
std::vector<uint32_t> restarts_; // Restart points
|
||||
int counter_; // Number of entries emitted since restart
|
||||
bool finished_; // Has Finish() been called?
|
||||
std::string last_key_;
|
||||
|
||||
// No copying allowed
|
||||
BlockBuilder(const BlockBuilder&);
|
||||
void operator=(const BlockBuilder&);
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
|
@ -16,7 +16,8 @@ static const size_t kFilterBaseLg = 11;
|
||||
static const size_t kFilterBase = 1 << kFilterBaseLg;
|
||||
|
||||
FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy)
|
||||
: policy_(policy) {}
|
||||
: policy_(policy) {
|
||||
}
|
||||
|
||||
void FilterBlockBuilder::StartBlock(uint64_t block_offset) {
|
||||
uint64_t filter_index = (block_offset / kFilterBase);
|
||||
@ -61,7 +62,7 @@ void FilterBlockBuilder::GenerateFilter() {
|
||||
tmp_keys_.resize(num_keys);
|
||||
for (size_t i = 0; i < num_keys; i++) {
|
||||
const char* base = keys_.data() + start_[i];
|
||||
size_t length = start_[i + 1] - start_[i];
|
||||
size_t length = start_[i+1] - start_[i];
|
||||
tmp_keys_[i] = Slice(base, length);
|
||||
}
|
||||
|
||||
@ -76,10 +77,14 @@ void FilterBlockBuilder::GenerateFilter() {
|
||||
|
||||
FilterBlockReader::FilterBlockReader(const FilterPolicy* policy,
|
||||
const Slice& contents)
|
||||
: policy_(policy), data_(nullptr), offset_(nullptr), num_(0), base_lg_(0) {
|
||||
: policy_(policy),
|
||||
data_(nullptr),
|
||||
offset_(nullptr),
|
||||
num_(0),
|
||||
base_lg_(0) {
|
||||
size_t n = contents.size();
|
||||
if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array
|
||||
base_lg_ = contents[n - 1];
|
||||
base_lg_ = contents[n-1];
|
||||
uint32_t last_word = DecodeFixed32(contents.data() + n - 5);
|
||||
if (last_word > n - 5) return;
|
||||
data_ = contents.data();
|
||||
@ -90,8 +95,8 @@ FilterBlockReader::FilterBlockReader(const FilterPolicy* policy,
|
||||
bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) {
|
||||
uint64_t index = block_offset >> base_lg_;
|
||||
if (index < num_) {
|
||||
uint32_t start = DecodeFixed32(offset_ + index * 4);
|
||||
uint32_t limit = DecodeFixed32(offset_ + index * 4 + 4);
|
||||
uint32_t start = DecodeFixed32(offset_ + index*4);
|
||||
uint32_t limit = DecodeFixed32(offset_ + index*4 + 4);
|
||||
if (start <= limit && limit <= static_cast<size_t>(offset_ - data_)) {
|
||||
Slice filter = Slice(data_ + start, limit - start);
|
||||
return policy_->KeyMayMatch(key, filter);
|
||||
@ -103,4 +108,4 @@ bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) {
|
||||
return true; // Errors are treated as potential matches
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
}
|
||||
|
@ -9,11 +9,10 @@
|
||||
#ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
|
||||
#define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "leveldb/slice.h"
|
||||
#include "util/hash.h"
|
||||
|
||||
@ -31,9 +30,6 @@ class FilterBlockBuilder {
|
||||
public:
|
||||
explicit FilterBlockBuilder(const FilterPolicy*);
|
||||
|
||||
FilterBlockBuilder(const FilterBlockBuilder&) = delete;
|
||||
FilterBlockBuilder& operator=(const FilterBlockBuilder&) = delete;
|
||||
|
||||
void StartBlock(uint64_t block_offset);
|
||||
void AddKey(const Slice& key);
|
||||
Slice Finish();
|
||||
@ -42,16 +38,20 @@ class FilterBlockBuilder {
|
||||
void GenerateFilter();
|
||||
|
||||
const FilterPolicy* policy_;
|
||||
std::string keys_; // Flattened key contents
|
||||
std::vector<size_t> start_; // Starting index in keys_ of each key
|
||||
std::string result_; // Filter data computed so far
|
||||
std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
|
||||
std::string keys_; // Flattened key contents
|
||||
std::vector<size_t> start_; // Starting index in keys_ of each key
|
||||
std::string result_; // Filter data computed so far
|
||||
std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
|
||||
std::vector<uint32_t> filter_offsets_;
|
||||
|
||||
// No copying allowed
|
||||
FilterBlockBuilder(const FilterBlockBuilder&);
|
||||
void operator=(const FilterBlockBuilder&);
|
||||
};
|
||||
|
||||
class FilterBlockReader {
|
||||
public:
|
||||
// REQUIRES: "contents" and *policy must stay live while *this is live.
|
||||
// REQUIRES: "contents" and *policy must stay live while *this is live.
|
||||
FilterBlockReader(const FilterPolicy* policy, const Slice& contents);
|
||||
bool KeyMayMatch(uint64_t block_offset, const Slice& key);
|
||||
|
||||
@ -63,6 +63,6 @@ class FilterBlockReader {
|
||||
size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
|
||||
};
|
||||
|
||||
} // namespace leveldb
|
||||
}
|
||||
|
||||
#endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
|
||||
|
@ -4,11 +4,11 @@
|
||||
|
||||
#include "table/filter_block.h"
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/hash.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace leveldb {
|
||||
@ -16,16 +16,18 @@ namespace leveldb {
|
||||
// For testing: emit an array with one hash value per key
|
||||
class TestHashFilter : public FilterPolicy {
|
||||
public:
|
||||
const char* Name() const override { return "TestHashFilter"; }
|
||||
virtual const char* Name() const {
|
||||
return "TestHashFilter";
|
||||
}
|
||||
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
for (int i = 0; i < n; i++) {
|
||||
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1);
|
||||
PutFixed32(dst, h);
|
||||
}
|
||||
}
|
||||
|
||||
bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
|
||||
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
|
||||
uint32_t h = Hash(key.data(), key.size(), 1);
|
||||
for (size_t i = 0; i + 4 <= filter.size(); i += 4) {
|
||||
if (h == DecodeFixed32(filter.data() + i)) {
|
||||
@ -36,12 +38,12 @@ class TestHashFilter : public FilterPolicy {
|
||||
}
|
||||
};
|
||||
|
||||
class FilterBlockTest : public testing::Test {
|
||||
class FilterBlockTest {
|
||||
public:
|
||||
TestHashFilter policy_;
|
||||
};
|
||||
|
||||
TEST_F(FilterBlockTest, EmptyBuilder) {
|
||||
TEST(FilterBlockTest, EmptyBuilder) {
|
||||
FilterBlockBuilder builder(&policy_);
|
||||
Slice block = builder.Finish();
|
||||
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block));
|
||||
@ -50,7 +52,7 @@ TEST_F(FilterBlockTest, EmptyBuilder) {
|
||||
ASSERT_TRUE(reader.KeyMayMatch(100000, "foo"));
|
||||
}
|
||||
|
||||
TEST_F(FilterBlockTest, SingleChunk) {
|
||||
TEST(FilterBlockTest, SingleChunk) {
|
||||
FilterBlockBuilder builder(&policy_);
|
||||
builder.StartBlock(100);
|
||||
builder.AddKey("foo");
|
||||
@ -67,11 +69,11 @@ TEST_F(FilterBlockTest, SingleChunk) {
|
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "box"));
|
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "hello"));
|
||||
ASSERT_TRUE(reader.KeyMayMatch(100, "foo"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(100, "missing"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(100, "other"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(100, "missing"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(100, "other"));
|
||||
}
|
||||
|
||||
TEST_F(FilterBlockTest, MultiChunk) {
|
||||
TEST(FilterBlockTest, MultiChunk) {
|
||||
FilterBlockBuilder builder(&policy_);
|
||||
|
||||
// First filter
|
||||
@ -97,26 +99,30 @@ TEST_F(FilterBlockTest, MultiChunk) {
|
||||
// Check first filter
|
||||
ASSERT_TRUE(reader.KeyMayMatch(0, "foo"));
|
||||
ASSERT_TRUE(reader.KeyMayMatch(2000, "bar"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(0, "box"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(0, "hello"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(0, "box"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(0, "hello"));
|
||||
|
||||
// Check second filter
|
||||
ASSERT_TRUE(reader.KeyMayMatch(3100, "box"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(3100, "foo"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(3100, "bar"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(3100, "hello"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello"));
|
||||
|
||||
// Check third filter (empty)
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(4100, "foo"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(4100, "bar"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(4100, "box"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(4100, "hello"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "box"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello"));
|
||||
|
||||
// Check last filter
|
||||
ASSERT_TRUE(reader.KeyMayMatch(9000, "box"));
|
||||
ASSERT_TRUE(reader.KeyMayMatch(9000, "hello"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(9000, "foo"));
|
||||
ASSERT_TRUE(!reader.KeyMayMatch(9000, "bar"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo"));
|
||||
ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar"));
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
|
@ -5,7 +5,6 @@
|
||||
#include "table/format.h"
|
||||
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "port/port.h"
|
||||
#include "table/block.h"
|
||||
#include "util/coding.h"
|
||||
@ -22,7 +21,8 @@ void BlockHandle::EncodeTo(std::string* dst) const {
|
||||
}
|
||||
|
||||
Status BlockHandle::DecodeFrom(Slice* input) {
|
||||
if (GetVarint64(input, &offset_) && GetVarint64(input, &size_)) {
|
||||
if (GetVarint64(input, &offset_) &&
|
||||
GetVarint64(input, &size_)) {
|
||||
return Status::OK();
|
||||
} else {
|
||||
return Status::Corruption("bad block handle");
|
||||
@ -41,10 +41,6 @@ void Footer::EncodeTo(std::string* dst) const {
|
||||
}
|
||||
|
||||
Status Footer::DecodeFrom(Slice* input) {
|
||||
if (input->size() < kEncodedLength) {
|
||||
return Status::Corruption("not an sstable (footer too short)");
|
||||
}
|
||||
|
||||
const char* magic_ptr = input->data() + kEncodedLength - 8;
|
||||
const uint32_t magic_lo = DecodeFixed32(magic_ptr);
|
||||
const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4);
|
||||
@ -66,8 +62,10 @@ Status Footer::DecodeFrom(Slice* input) {
|
||||
return result;
|
||||
}
|
||||
|
||||
Status ReadBlock(RandomAccessFile* file, const ReadOptions& options,
|
||||
const BlockHandle& handle, BlockContents* result) {
|
||||
Status ReadBlock(RandomAccessFile* file,
|
||||
const ReadOptions& options,
|
||||
const BlockHandle& handle,
|
||||
BlockContents* result) {
|
||||
result->data = Slice();
|
||||
result->cachable = false;
|
||||
result->heap_allocated = false;
|
||||
@ -88,7 +86,7 @@ Status ReadBlock(RandomAccessFile* file, const ReadOptions& options,
|
||||
}
|
||||
|
||||
// Check the crc of the type and the block contents
|
||||
const char* data = contents.data(); // Pointer to where Read put the data
|
||||
const char* data = contents.data(); // Pointer to where Read put the data
|
||||
if (options.verify_checksums) {
|
||||
const uint32_t crc = crc32c::Unmask(DecodeFixed32(data + n + 1));
|
||||
const uint32_t actual = crc32c::Value(data, n + 1);
|
||||
@ -121,31 +119,13 @@ Status ReadBlock(RandomAccessFile* file, const ReadOptions& options,
|
||||
size_t ulength = 0;
|
||||
if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) {
|
||||
delete[] buf;
|
||||
return Status::Corruption("corrupted snappy compressed block length");
|
||||
return Status::Corruption("corrupted compressed block contents");
|
||||
}
|
||||
char* ubuf = new char[ulength];
|
||||
if (!port::Snappy_Uncompress(data, n, ubuf)) {
|
||||
delete[] buf;
|
||||
delete[] ubuf;
|
||||
return Status::Corruption("corrupted snappy compressed block contents");
|
||||
}
|
||||
delete[] buf;
|
||||
result->data = Slice(ubuf, ulength);
|
||||
result->heap_allocated = true;
|
||||
result->cachable = true;
|
||||
break;
|
||||
}
|
||||
case kZstdCompression: {
|
||||
size_t ulength = 0;
|
||||
if (!port::Zstd_GetUncompressedLength(data, n, &ulength)) {
|
||||
delete[] buf;
|
||||
return Status::Corruption("corrupted zstd compressed block length");
|
||||
}
|
||||
char* ubuf = new char[ulength];
|
||||
if (!port::Zstd_Uncompress(data, n, ubuf)) {
|
||||
delete[] buf;
|
||||
delete[] ubuf;
|
||||
return Status::Corruption("corrupted zstd compressed block contents");
|
||||
return Status::Corruption("corrupted compressed block contents");
|
||||
}
|
||||
delete[] buf;
|
||||
result->data = Slice(ubuf, ulength);
|
||||
|
@ -5,9 +5,8 @@
|
||||
#ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_TABLE_FORMAT_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include <stdint.h>
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/table_builder.h"
|
||||
@ -22,9 +21,6 @@ struct ReadOptions;
|
||||
// block or a meta block.
|
||||
class BlockHandle {
|
||||
public:
|
||||
// Maximum encoding length of a BlockHandle
|
||||
enum { kMaxEncodedLength = 10 + 10 };
|
||||
|
||||
BlockHandle();
|
||||
|
||||
// The offset of the block in the file.
|
||||
@ -38,6 +34,9 @@ class BlockHandle {
|
||||
void EncodeTo(std::string* dst) const;
|
||||
Status DecodeFrom(Slice* input);
|
||||
|
||||
// Maximum encoding length of a BlockHandle
|
||||
enum { kMaxEncodedLength = 10 + 10 };
|
||||
|
||||
private:
|
||||
uint64_t offset_;
|
||||
uint64_t size_;
|
||||
@ -47,24 +46,30 @@ class BlockHandle {
|
||||
// end of every table file.
|
||||
class Footer {
|
||||
public:
|
||||
// Encoded length of a Footer. Note that the serialization of a
|
||||
// Footer will always occupy exactly this many bytes. It consists
|
||||
// of two block handles and a magic number.
|
||||
enum { kEncodedLength = 2 * BlockHandle::kMaxEncodedLength + 8 };
|
||||
|
||||
Footer() = default;
|
||||
Footer() { }
|
||||
|
||||
// The block handle for the metaindex block of the table
|
||||
const BlockHandle& metaindex_handle() const { return metaindex_handle_; }
|
||||
void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; }
|
||||
|
||||
// The block handle for the index block of the table
|
||||
const BlockHandle& index_handle() const { return index_handle_; }
|
||||
void set_index_handle(const BlockHandle& h) { index_handle_ = h; }
|
||||
const BlockHandle& index_handle() const {
|
||||
return index_handle_;
|
||||
}
|
||||
void set_index_handle(const BlockHandle& h) {
|
||||
index_handle_ = h;
|
||||
}
|
||||
|
||||
void EncodeTo(std::string* dst) const;
|
||||
Status DecodeFrom(Slice* input);
|
||||
|
||||
// Encoded length of a Footer. Note that the serialization of a
|
||||
// Footer will always occupy exactly this many bytes. It consists
|
||||
// of two block handles and a magic number.
|
||||
enum {
|
||||
kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8
|
||||
};
|
||||
|
||||
private:
|
||||
BlockHandle metaindex_handle_;
|
||||
BlockHandle index_handle_;
|
||||
@ -86,13 +91,17 @@ struct BlockContents {
|
||||
|
||||
// Read the block identified by "handle" from "file". On failure
|
||||
// return non-OK. On success fill *result and return OK.
|
||||
Status ReadBlock(RandomAccessFile* file, const ReadOptions& options,
|
||||
const BlockHandle& handle, BlockContents* result);
|
||||
Status ReadBlock(RandomAccessFile* file,
|
||||
const ReadOptions& options,
|
||||
const BlockHandle& handle,
|
||||
BlockContents* result);
|
||||
|
||||
// Implementation details follow. Clients should ignore,
|
||||
|
||||
inline BlockHandle::BlockHandle()
|
||||
: offset_(~static_cast<uint64_t>(0)), size_(~static_cast<uint64_t>(0)) {}
|
||||
: offset_(~static_cast<uint64_t>(0)),
|
||||
size_(~static_cast<uint64_t>(0)) {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
@ -14,7 +14,7 @@ Iterator::Iterator() {
|
||||
Iterator::~Iterator() {
|
||||
if (!cleanup_head_.IsEmpty()) {
|
||||
cleanup_head_.Run();
|
||||
for (CleanupNode* node = cleanup_head_.next; node != nullptr;) {
|
||||
for (CleanupNode* node = cleanup_head_.next; node != nullptr; ) {
|
||||
node->Run();
|
||||
CleanupNode* next_node = node->next;
|
||||
delete node;
|
||||
@ -42,23 +42,17 @@ namespace {
|
||||
|
||||
class EmptyIterator : public Iterator {
|
||||
public:
|
||||
EmptyIterator(const Status& s) : status_(s) {}
|
||||
EmptyIterator(const Status& s) : status_(s) { }
|
||||
~EmptyIterator() override = default;
|
||||
|
||||
bool Valid() const override { return false; }
|
||||
void Seek(const Slice& target) override {}
|
||||
void SeekToFirst() override {}
|
||||
void SeekToLast() override {}
|
||||
void Seek(const Slice& target) override { }
|
||||
void SeekToFirst() override { }
|
||||
void SeekToLast() override { }
|
||||
void Next() override { assert(false); }
|
||||
void Prev() override { assert(false); }
|
||||
Slice key() const override {
|
||||
assert(false);
|
||||
return Slice();
|
||||
}
|
||||
Slice value() const override {
|
||||
assert(false);
|
||||
return Slice();
|
||||
}
|
||||
Slice key() const override { assert(false); return Slice(); }
|
||||
Slice value() const override { assert(false); return Slice(); }
|
||||
Status status() const override { return status_; }
|
||||
|
||||
private:
|
||||
@ -67,7 +61,9 @@ class EmptyIterator : public Iterator {
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
Iterator* NewEmptyIterator() { return new EmptyIterator(Status::OK()); }
|
||||
Iterator* NewEmptyIterator() {
|
||||
return new EmptyIterator(Status::OK());
|
||||
}
|
||||
|
||||
Iterator* NewErrorIterator(const Status& status) {
|
||||
return new EmptyIterator(status);
|
||||
|
@ -16,8 +16,10 @@ namespace leveldb {
|
||||
// cache locality.
|
||||
class IteratorWrapper {
|
||||
public:
|
||||
IteratorWrapper() : iter_(nullptr), valid_(false) {}
|
||||
explicit IteratorWrapper(Iterator* iter) : iter_(nullptr) { Set(iter); }
|
||||
IteratorWrapper(): iter_(nullptr), valid_(false) { }
|
||||
explicit IteratorWrapper(Iterator* iter): iter_(nullptr) {
|
||||
Set(iter);
|
||||
}
|
||||
~IteratorWrapper() { delete iter_; }
|
||||
Iterator* iter() const { return iter_; }
|
||||
|
||||
@ -33,46 +35,18 @@ class IteratorWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Iterator interface methods
|
||||
bool Valid() const { return valid_; }
|
||||
Slice key() const {
|
||||
assert(Valid());
|
||||
return key_;
|
||||
}
|
||||
Slice value() const {
|
||||
assert(Valid());
|
||||
return iter_->value();
|
||||
}
|
||||
bool Valid() const { return valid_; }
|
||||
Slice key() const { assert(Valid()); return key_; }
|
||||
Slice value() const { assert(Valid()); return iter_->value(); }
|
||||
// Methods below require iter() != nullptr
|
||||
Status status() const {
|
||||
assert(iter_);
|
||||
return iter_->status();
|
||||
}
|
||||
void Next() {
|
||||
assert(iter_);
|
||||
iter_->Next();
|
||||
Update();
|
||||
}
|
||||
void Prev() {
|
||||
assert(iter_);
|
||||
iter_->Prev();
|
||||
Update();
|
||||
}
|
||||
void Seek(const Slice& k) {
|
||||
assert(iter_);
|
||||
iter_->Seek(k);
|
||||
Update();
|
||||
}
|
||||
void SeekToFirst() {
|
||||
assert(iter_);
|
||||
iter_->SeekToFirst();
|
||||
Update();
|
||||
}
|
||||
void SeekToLast() {
|
||||
assert(iter_);
|
||||
iter_->SeekToLast();
|
||||
Update();
|
||||
}
|
||||
Status status() const { assert(iter_); return iter_->status(); }
|
||||
void Next() { assert(iter_); iter_->Next(); Update(); }
|
||||
void Prev() { assert(iter_); iter_->Prev(); Update(); }
|
||||
void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); }
|
||||
void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); }
|
||||
void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); }
|
||||
|
||||
private:
|
||||
void Update() {
|
||||
|
@ -24,11 +24,15 @@ class MergingIterator : public Iterator {
|
||||
}
|
||||
}
|
||||
|
||||
~MergingIterator() override { delete[] children_; }
|
||||
virtual ~MergingIterator() {
|
||||
delete[] children_;
|
||||
}
|
||||
|
||||
bool Valid() const override { return (current_ != nullptr); }
|
||||
virtual bool Valid() const {
|
||||
return (current_ != nullptr);
|
||||
}
|
||||
|
||||
void SeekToFirst() override {
|
||||
virtual void SeekToFirst() {
|
||||
for (int i = 0; i < n_; i++) {
|
||||
children_[i].SeekToFirst();
|
||||
}
|
||||
@ -36,7 +40,7 @@ class MergingIterator : public Iterator {
|
||||
direction_ = kForward;
|
||||
}
|
||||
|
||||
void SeekToLast() override {
|
||||
virtual void SeekToLast() {
|
||||
for (int i = 0; i < n_; i++) {
|
||||
children_[i].SeekToLast();
|
||||
}
|
||||
@ -44,7 +48,7 @@ class MergingIterator : public Iterator {
|
||||
direction_ = kReverse;
|
||||
}
|
||||
|
||||
void Seek(const Slice& target) override {
|
||||
virtual void Seek(const Slice& target) {
|
||||
for (int i = 0; i < n_; i++) {
|
||||
children_[i].Seek(target);
|
||||
}
|
||||
@ -52,7 +56,7 @@ class MergingIterator : public Iterator {
|
||||
direction_ = kForward;
|
||||
}
|
||||
|
||||
void Next() override {
|
||||
virtual void Next() {
|
||||
assert(Valid());
|
||||
|
||||
// Ensure that all children are positioned after key().
|
||||
@ -78,7 +82,7 @@ class MergingIterator : public Iterator {
|
||||
FindSmallest();
|
||||
}
|
||||
|
||||
void Prev() override {
|
||||
virtual void Prev() {
|
||||
assert(Valid());
|
||||
|
||||
// Ensure that all children are positioned before key().
|
||||
@ -107,17 +111,17 @@ class MergingIterator : public Iterator {
|
||||
FindLargest();
|
||||
}
|
||||
|
||||
Slice key() const override {
|
||||
virtual Slice key() const {
|
||||
assert(Valid());
|
||||
return current_->key();
|
||||
}
|
||||
|
||||
Slice value() const override {
|
||||
virtual Slice value() const {
|
||||
assert(Valid());
|
||||
return current_->value();
|
||||
}
|
||||
|
||||
Status status() const override {
|
||||
virtual Status status() const {
|
||||
Status status;
|
||||
for (int i = 0; i < n_; i++) {
|
||||
status = children_[i].status();
|
||||
@ -129,9 +133,6 @@ class MergingIterator : public Iterator {
|
||||
}
|
||||
|
||||
private:
|
||||
// Which direction is the iterator moving?
|
||||
enum Direction { kForward, kReverse };
|
||||
|
||||
void FindSmallest();
|
||||
void FindLargest();
|
||||
|
||||
@ -142,6 +143,12 @@ class MergingIterator : public Iterator {
|
||||
IteratorWrapper* children_;
|
||||
int n_;
|
||||
IteratorWrapper* current_;
|
||||
|
||||
// Which direction is the iterator moving?
|
||||
enum Direction {
|
||||
kForward,
|
||||
kReverse
|
||||
};
|
||||
Direction direction_;
|
||||
};
|
||||
|
||||
@ -162,7 +169,7 @@ void MergingIterator::FindSmallest() {
|
||||
|
||||
void MergingIterator::FindLargest() {
|
||||
IteratorWrapper* largest = nullptr;
|
||||
for (int i = n_ - 1; i >= 0; i--) {
|
||||
for (int i = n_-1; i >= 0; i--) {
|
||||
IteratorWrapper* child = &children_[i];
|
||||
if (child->Valid()) {
|
||||
if (largest == nullptr) {
|
||||
@ -176,15 +183,14 @@ void MergingIterator::FindLargest() {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children,
|
||||
int n) {
|
||||
Iterator* NewMergingIterator(const Comparator* cmp, Iterator** list, int n) {
|
||||
assert(n >= 0);
|
||||
if (n == 0) {
|
||||
return NewEmptyIterator();
|
||||
} else if (n == 1) {
|
||||
return children[0];
|
||||
return list[0];
|
||||
} else {
|
||||
return new MergingIterator(comparator, children, n);
|
||||
return new MergingIterator(cmp, list, n);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -18,8 +18,8 @@ class Iterator;
|
||||
// key is present in K child iterators, it will be yielded K times.
|
||||
//
|
||||
// REQUIRES: n >= 0
|
||||
Iterator* NewMergingIterator(const Comparator* comparator, Iterator** children,
|
||||
int n);
|
||||
Iterator* NewMergingIterator(
|
||||
const Comparator* comparator, Iterator** children, int n);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
|
@ -20,7 +20,7 @@ namespace leveldb {
|
||||
struct Table::Rep {
|
||||
~Rep() {
|
||||
delete filter;
|
||||
delete[] filter_data;
|
||||
delete [] filter_data;
|
||||
delete index_block;
|
||||
}
|
||||
|
||||
@ -35,8 +35,10 @@ struct Table::Rep {
|
||||
Block* index_block;
|
||||
};
|
||||
|
||||
Status Table::Open(const Options& options, RandomAccessFile* file,
|
||||
uint64_t size, Table** table) {
|
||||
Status Table::Open(const Options& options,
|
||||
RandomAccessFile* file,
|
||||
uint64_t size,
|
||||
Table** table) {
|
||||
*table = nullptr;
|
||||
if (size < Footer::kEncodedLength) {
|
||||
return Status::Corruption("file is too short to be an sstable");
|
||||
@ -54,11 +56,13 @@ Status Table::Open(const Options& options, RandomAccessFile* file,
|
||||
|
||||
// Read the index block
|
||||
BlockContents index_block_contents;
|
||||
ReadOptions opt;
|
||||
if (options.paranoid_checks) {
|
||||
opt.verify_checksums = true;
|
||||
if (s.ok()) {
|
||||
ReadOptions opt;
|
||||
if (options.paranoid_checks) {
|
||||
opt.verify_checksums = true;
|
||||
}
|
||||
s = ReadBlock(file, opt, footer.index_handle(), &index_block_contents);
|
||||
}
|
||||
s = ReadBlock(file, opt, footer.index_handle(), &index_block_contents);
|
||||
|
||||
if (s.ok()) {
|
||||
// We've successfully read the footer and the index block: we're
|
||||
@ -126,12 +130,14 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
|
||||
return;
|
||||
}
|
||||
if (block.heap_allocated) {
|
||||
rep_->filter_data = block.data.data(); // Will need to delete later
|
||||
rep_->filter_data = block.data.data(); // Will need to delete later
|
||||
}
|
||||
rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data);
|
||||
}
|
||||
|
||||
Table::~Table() { delete rep_; }
|
||||
Table::~Table() {
|
||||
delete rep_;
|
||||
}
|
||||
|
||||
static void DeleteBlock(void* arg, void* ignored) {
|
||||
delete reinterpret_cast<Block*>(arg);
|
||||
@ -150,7 +156,8 @@ static void ReleaseBlock(void* arg, void* h) {
|
||||
|
||||
// Convert an index iterator value (i.e., an encoded BlockHandle)
|
||||
// into an iterator over the contents of the corresponding block.
|
||||
Iterator* Table::BlockReader(void* arg, const ReadOptions& options,
|
||||
Iterator* Table::BlockReader(void* arg,
|
||||
const ReadOptions& options,
|
||||
const Slice& index_value) {
|
||||
Table* table = reinterpret_cast<Table*>(arg);
|
||||
Cache* block_cache = table->rep_->options.block_cache;
|
||||
@ -168,7 +175,7 @@ Iterator* Table::BlockReader(void* arg, const ReadOptions& options,
|
||||
if (block_cache != nullptr) {
|
||||
char cache_key_buffer[16];
|
||||
EncodeFixed64(cache_key_buffer, table->rep_->cache_id);
|
||||
EncodeFixed64(cache_key_buffer + 8, handle.offset());
|
||||
EncodeFixed64(cache_key_buffer+8, handle.offset());
|
||||
Slice key(cache_key_buffer, sizeof(cache_key_buffer));
|
||||
cache_handle = block_cache->Lookup(key);
|
||||
if (cache_handle != nullptr) {
|
||||
@ -178,8 +185,8 @@ Iterator* Table::BlockReader(void* arg, const ReadOptions& options,
|
||||
if (s.ok()) {
|
||||
block = new Block(contents);
|
||||
if (contents.cachable && options.fill_cache) {
|
||||
cache_handle = block_cache->Insert(key, block, block->size(),
|
||||
&DeleteCachedBlock);
|
||||
cache_handle = block_cache->Insert(
|
||||
key, block, block->size(), &DeleteCachedBlock);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -211,9 +218,9 @@ Iterator* Table::NewIterator(const ReadOptions& options) const {
|
||||
&Table::BlockReader, const_cast<Table*>(this), options);
|
||||
}
|
||||
|
||||
Status Table::InternalGet(const ReadOptions& options, const Slice& k, void* arg,
|
||||
void (*handle_result)(void*, const Slice&,
|
||||
const Slice&)) {
|
||||
Status Table::InternalGet(const ReadOptions& options, const Slice& k,
|
||||
void* arg,
|
||||
void (*saver)(void*, const Slice&, const Slice&)) {
|
||||
Status s;
|
||||
Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
|
||||
iiter->Seek(k);
|
||||
@ -221,14 +228,15 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k, void* arg,
|
||||
Slice handle_value = iiter->value();
|
||||
FilterBlockReader* filter = rep_->filter;
|
||||
BlockHandle handle;
|
||||
if (filter != nullptr && handle.DecodeFrom(&handle_value).ok() &&
|
||||
if (filter != nullptr &&
|
||||
handle.DecodeFrom(&handle_value).ok() &&
|
||||
!filter->KeyMayMatch(handle.offset(), k)) {
|
||||
// Not found
|
||||
} else {
|
||||
Iterator* block_iter = BlockReader(this, options, iiter->value());
|
||||
block_iter->Seek(k);
|
||||
if (block_iter->Valid()) {
|
||||
(*handle_result)(arg, block_iter->key(), block_iter->value());
|
||||
(*saver)(arg, block_iter->key(), block_iter->value());
|
||||
}
|
||||
s = block_iter->status();
|
||||
delete block_iter;
|
||||
@ -241,6 +249,7 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k, void* arg,
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
uint64_t Table::ApproximateOffsetOf(const Slice& key) const {
|
||||
Iterator* index_iter =
|
||||
rep_->index_block->NewIterator(rep_->options.comparator);
|
||||
|
@ -4,8 +4,7 @@
|
||||
|
||||
#include "leveldb/table_builder.h"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
#include <assert.h>
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
@ -19,22 +18,6 @@
|
||||
namespace leveldb {
|
||||
|
||||
struct TableBuilder::Rep {
|
||||
Rep(const Options& opt, WritableFile* f)
|
||||
: options(opt),
|
||||
index_block_options(opt),
|
||||
file(f),
|
||||
offset(0),
|
||||
data_block(&options),
|
||||
index_block(&index_block_options),
|
||||
num_entries(0),
|
||||
closed(false),
|
||||
filter_block(opt.filter_policy == nullptr
|
||||
? nullptr
|
||||
: new FilterBlockBuilder(opt.filter_policy)),
|
||||
pending_index_entry(false) {
|
||||
index_block_options.block_restart_interval = 1;
|
||||
}
|
||||
|
||||
Options options;
|
||||
Options index_block_options;
|
||||
WritableFile* file;
|
||||
@ -44,7 +27,7 @@ struct TableBuilder::Rep {
|
||||
BlockBuilder index_block;
|
||||
std::string last_key;
|
||||
int64_t num_entries;
|
||||
bool closed; // Either Finish() or Abandon() has been called.
|
||||
bool closed; // Either Finish() or Abandon() has been called.
|
||||
FilterBlockBuilder* filter_block;
|
||||
|
||||
// We do not emit the index entry for a block until we have seen the
|
||||
@ -60,6 +43,21 @@ struct TableBuilder::Rep {
|
||||
BlockHandle pending_handle; // Handle to add to index block
|
||||
|
||||
std::string compressed_output;
|
||||
|
||||
Rep(const Options& opt, WritableFile* f)
|
||||
: options(opt),
|
||||
index_block_options(opt),
|
||||
file(f),
|
||||
offset(0),
|
||||
data_block(&options),
|
||||
index_block(&index_block_options),
|
||||
num_entries(0),
|
||||
closed(false),
|
||||
filter_block(opt.filter_policy == nullptr ? nullptr
|
||||
: new FilterBlockBuilder(opt.filter_policy)),
|
||||
pending_index_entry(false) {
|
||||
index_block_options.block_restart_interval = 1;
|
||||
}
|
||||
};
|
||||
|
||||
TableBuilder::TableBuilder(const Options& options, WritableFile* file)
|
||||
@ -168,21 +166,6 @@ void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case kZstdCompression: {
|
||||
std::string* compressed = &r->compressed_output;
|
||||
if (port::Zstd_Compress(r->options.zstd_compression_level, raw.data(),
|
||||
raw.size(), compressed) &&
|
||||
compressed->size() < raw.size() - (raw.size() / 8u)) {
|
||||
block_contents = *compressed;
|
||||
} else {
|
||||
// Zstd not supported, or compressed less than 12.5%, so just
|
||||
// store uncompressed form
|
||||
block_contents = raw;
|
||||
type = kNoCompression;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
WriteRawBlock(block_contents, type, handle);
|
||||
r->compressed_output.clear();
|
||||
@ -190,7 +173,8 @@ void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) {
|
||||
}
|
||||
|
||||
void TableBuilder::WriteRawBlock(const Slice& block_contents,
|
||||
CompressionType type, BlockHandle* handle) {
|
||||
CompressionType type,
|
||||
BlockHandle* handle) {
|
||||
Rep* r = rep_;
|
||||
handle->set_offset(r->offset);
|
||||
handle->set_size(block_contents.size());
|
||||
@ -200,7 +184,7 @@ void TableBuilder::WriteRawBlock(const Slice& block_contents,
|
||||
trailer[0] = type;
|
||||
uint32_t crc = crc32c::Value(block_contents.data(), block_contents.size());
|
||||
crc = crc32c::Extend(crc, trailer, 1); // Extend crc to cover block type
|
||||
EncodeFixed32(trailer + 1, crc32c::Mask(crc));
|
||||
EncodeFixed32(trailer+1, crc32c::Mask(crc));
|
||||
r->status = r->file->Append(Slice(trailer, kBlockTrailerSize));
|
||||
if (r->status.ok()) {
|
||||
r->offset += block_contents.size() + kBlockTrailerSize;
|
||||
@ -208,7 +192,9 @@ void TableBuilder::WriteRawBlock(const Slice& block_contents,
|
||||
}
|
||||
}
|
||||
|
||||
Status TableBuilder::status() const { return rep_->status; }
|
||||
Status TableBuilder::status() const {
|
||||
return rep_->status;
|
||||
}
|
||||
|
||||
Status TableBuilder::Finish() {
|
||||
Rep* r = rep_;
|
||||
@ -273,8 +259,12 @@ void TableBuilder::Abandon() {
|
||||
r->closed = true;
|
||||
}
|
||||
|
||||
uint64_t TableBuilder::NumEntries() const { return rep_->num_entries; }
|
||||
uint64_t TableBuilder::NumEntries() const {
|
||||
return rep_->num_entries;
|
||||
}
|
||||
|
||||
uint64_t TableBuilder::FileSize() const { return rep_->offset; }
|
||||
uint64_t TableBuilder::FileSize() const {
|
||||
return rep_->offset;
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user