Remove static initializer; fix endian-ness detection; fix build on

various platforms; improve android port speed.

Avoid static initializer by using a new portability interface for
thread-safe lazy initialization.  Custom ports will need to be
extended to implement InitOnce/OnceType/LEVELDB_ONCE_INIT.

Fix endian-ness detection (fixes Powerpc builds).

Build related fixes:
- Support platforms that have unversioned shared libraries.
- Fix IOS build rules.

Android improvements
- Speed up atomic pointers
- Share more code with port_posix.

Do not spin in a tight loop attempting compactions if the file system
is inaccessible (e.g., if kerberos tickets have expired or if it is out
of space).
This commit is contained in:
Sanjay Ghemawat 2012-05-30 09:45:46 -07:00
parent 85584d497e
commit 075a35a6d3
18 changed files with 283 additions and 62 deletions

View File

@ -3,8 +3,6 @@
# found in the LICENSE file. See the AUTHORS file for names of contributors. # found in the LICENSE file. See the AUTHORS file for names of contributors.
# Inherit some settings from environment variables, if available # Inherit some settings from environment variables, if available
CXX ?= g++
CC ?= gcc
INSTALL_PATH ?= $(CURDIR) INSTALL_PATH ?= $(CURDIR)
#----------------------------------------------- #-----------------------------------------------
@ -63,6 +61,13 @@ default: all
# Should we build shared libraries? # Should we build shared libraries?
ifneq ($(PLATFORM_SHARED_EXT),) ifneq ($(PLATFORM_SHARED_EXT),)
ifneq ($(PLATFORM_SHARED_VERSIONED),true)
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1)
SHARED3 = $(SHARED1)
SHARED = $(SHARED1)
else
# Update db.h if you change these. # Update db.h if you change these.
SHARED_MAJOR = 1 SHARED_MAJOR = 1
SHARED_MINOR = 4 SHARED_MINOR = 4
@ -70,14 +75,17 @@ SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED2 = $(SHARED1).$(SHARED_MAJOR)
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
SHARED = $(SHARED1) $(SHARED2) $(SHARED3) SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
$(SHARED3):
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(INSTALL_PATH)/$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
$(SHARED2): $(SHARED3)
ln -fs $(SHARED3) $(SHARED2)
$(SHARED1): $(SHARED3) $(SHARED1): $(SHARED3)
ln -fs $(SHARED3) $(SHARED1) ln -fs $(SHARED3) $(SHARED1)
$(SHARED2): $(SHARED3)
ln -fs $(SHARED3) $(SHARED2)
endif endif
$(SHARED3):
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
endif # PLATFORM_SHARED_EXT
all: $(SHARED) $(LIBRARY) all: $(SHARED) $(LIBRARY)
check: all $(PROGRAMS) $(TESTS) check: all $(PROGRAMS) $(TESTS)
@ -164,9 +172,10 @@ memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHAR
ifeq ($(PLATFORM), IOS) ifeq ($(PLATFORM), IOS)
# For iOS, create universal object files to be used on both the simulator and # For iOS, create universal object files to be used on both the simulator and
# a device. # a device.
SIMULATORROOT=/Developer/Platforms/iPhoneSimulator.platform/Developer PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms
DEVICEROOT=/Developer/Platforms/iPhoneOS.platform/Developer SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer
IOSVERSION=$(shell defaults read /Developer/Platforms/iPhoneOS.platform/version CFBundleShortVersionString) DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer
IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString)
.cc.o: .cc.o:
mkdir -p ios-x86/$(dir $@) mkdir -p ios-x86/$(dir $@)

View File

@ -4,18 +4,27 @@
# argument, which in turn gets read while processing Makefile. # argument, which in turn gets read while processing Makefile.
# #
# The output will set the following variables: # The output will set the following variables:
# CC C Compiler path
# CXX C++ Compiler path
# PLATFORM_LDFLAGS Linker flags # PLATFORM_LDFLAGS Linker flags
# PLATFORM_SHARED_EXT Extension for shared libraries # PLATFORM_SHARED_EXT Extension for shared libraries
# PLATFORM_SHARED_LDFLAGS Flags for building shared library # PLATFORM_SHARED_LDFLAGS Flags for building shared library
# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library # PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
# PLATFORM_CCFLAGS C compiler flags # PLATFORM_CCFLAGS C compiler flags
# PLATFORM_CXXFLAGS C++ compiler flags. Will contain: # PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
# -DLEVELDB_PLATFORM_POSIX if cstdatomic is present # PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned
# -DLEVELDB_PLATFORM_NOATOMIC if it is not # shared libraries, empty otherwise.
#
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
#
# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present
# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
# -DSNAPPY if the Snappy library is present
#
OUTPUT=$1 OUTPUT=$1
if test -z "$OUTPUT"; then if test -z "$OUTPUT"; then
echo "usage: $0 <output-filename>" echo "usage: $0 <output-filename>" >&2
exit 1 exit 1
fi fi
@ -23,6 +32,10 @@ fi
rm -f $OUTPUT rm -f $OUTPUT
touch $OUTPUT touch $OUTPUT
if test -z "$CC"; then
CC=cc
fi
if test -z "$CXX"; then if test -z "$CXX"; then
CXX=g++ CXX=g++
fi fi
@ -33,12 +46,14 @@ if test -z "$TARGET_OS"; then
fi fi
COMMON_FLAGS= COMMON_FLAGS=
CROSS_COMPILE=
PLATFORM_CCFLAGS= PLATFORM_CCFLAGS=
PLATFORM_CXXFLAGS= PLATFORM_CXXFLAGS=
PLATFORM_LDFLAGS= PLATFORM_LDFLAGS=
PLATFORM_SHARED_EXT="so" PLATFORM_SHARED_EXT="so"
PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl," PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
PLATFORM_SHARED_CFLAGS="-fPIC" PLATFORM_SHARED_CFLAGS="-fPIC"
PLATFORM_SHARED_VERSIONED=true
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
case "$TARGET_OS" in case "$TARGET_OS" in
@ -86,13 +101,14 @@ case "$TARGET_OS" in
PORT_FILE=port/port_posix.cc PORT_FILE=port/port_posix.cc
;; ;;
OS_ANDROID_CROSSCOMPILE) OS_ANDROID_CROSSCOMPILE)
PLATFORM="$TARGET_OS" PLATFORM=OS_ANDROID
COMMON_FLAGS="" COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
PLATFORM_LDFLAGS="" PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
PORT_FILE=port/port_android.cc PORT_FILE=port/port_posix.cc
CROSS_COMPILE=true
;; ;;
*) *)
echo "Unknown platform!" echo "Unknown platform!" >&2
exit 1 exit 1
esac esac
@ -112,7 +128,7 @@ set +f # re-enable globbing
echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
if [ "$PLATFORM" = "OS_ANDROID_CROSSCOMPILE" ]; then if [ "$CROSS_COMPILE" = "true" ]; then
# Cross-compiling; do not try any compilation tests. # Cross-compiling; do not try any compilation tests.
true true
else else
@ -151,6 +167,8 @@ fi
PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
echo "CC=$CC" >> $OUTPUT
echo "CXX=$CXX" >> $OUTPUT
echo "PLATFORM=$PLATFORM" >> $OUTPUT echo "PLATFORM=$PLATFORM" >> $OUTPUT
echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
@ -158,3 +176,4 @@ echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT

View File

@ -19,6 +19,13 @@ static void StartPhase(const char* name) {
phase = name; phase = name;
} }
static const char* GetTempDir(void) {
const char* ret = getenv("TEST_TMPDIR");
if (ret == NULL || ret[0] == '\0')
ret = "/tmp";
return ret;
}
#define CheckNoError(err) \ #define CheckNoError(err) \
if ((err) != NULL) { \ if ((err) != NULL) { \
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
@ -158,7 +165,9 @@ int main(int argc, char** argv) {
char* err = NULL; char* err = NULL;
int run = -1; int run = -1;
snprintf(dbname, sizeof(dbname), "/tmp/leveldb_c_test-%d", snprintf(dbname, sizeof(dbname),
"%s/leveldb_c_test-%d",
GetTempDir(),
((int) geteuid())); ((int) geteuid()));
StartPhase("create_objects"); StartPhase("create_objects");

View File

@ -100,7 +100,7 @@ static int FLAGS_bloom_bits = -1;
static bool FLAGS_use_existing_db = false; static bool FLAGS_use_existing_db = false;
// Use the db with the following name. // Use the db with the following name.
static const char* FLAGS_db = "/tmp/dbbench"; static const char* FLAGS_db = NULL;
namespace leveldb { namespace leveldb {
@ -925,6 +925,7 @@ class Benchmark {
int main(int argc, char** argv) { int main(int argc, char** argv) {
FLAGS_write_buffer_size = leveldb::Options().write_buffer_size; FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
FLAGS_open_files = leveldb::Options().max_open_files; FLAGS_open_files = leveldb::Options().max_open_files;
std::string default_db_path;
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
double d; double d;
@ -964,6 +965,13 @@ int main(int argc, char** argv) {
} }
} }
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db == NULL) {
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench";
FLAGS_db = default_db_path.c_str();
}
leveldb::Benchmark benchmark; leveldb::Benchmark benchmark;
benchmark.Run(); benchmark.Run();
return 0; return 0;

View File

@ -608,8 +608,21 @@ void DBImpl::BackgroundCall() {
MutexLock l(&mutex_); MutexLock l(&mutex_);
assert(bg_compaction_scheduled_); assert(bg_compaction_scheduled_);
if (!shutting_down_.Acquire_Load()) { if (!shutting_down_.Acquire_Load()) {
BackgroundCompaction(); Status s = BackgroundCompaction();
if (!s.ok()) {
// Wait a little bit before retrying background compaction in
// case this is an environmental problem and we do not want to
// chew up resources for failed compactions for the duration of
// the problem.
bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
Log(options_.info_log, "Waiting after background compaction error: %s",
s.ToString().c_str());
mutex_.Unlock();
env_->SleepForMicroseconds(1000000);
mutex_.Lock();
} }
}
bg_compaction_scheduled_ = false; bg_compaction_scheduled_ = false;
// Previous compaction may have produced too many files in a level, // Previous compaction may have produced too many files in a level,
@ -618,12 +631,11 @@ void DBImpl::BackgroundCall() {
bg_cv_.SignalAll(); bg_cv_.SignalAll();
} }
void DBImpl::BackgroundCompaction() { Status DBImpl::BackgroundCompaction() {
mutex_.AssertHeld(); mutex_.AssertHeld();
if (imm_ != NULL) { if (imm_ != NULL) {
CompactMemTable(); return CompactMemTable();
return;
} }
Compaction* c; Compaction* c;
@ -698,6 +710,7 @@ void DBImpl::BackgroundCompaction() {
} }
manual_compaction_ = NULL; manual_compaction_ = NULL;
} }
return status;
} }
void DBImpl::CleanupCompaction(CompactionState* compact) { void DBImpl::CleanupCompaction(CompactionState* compact) {
@ -1263,6 +1276,8 @@ Status DBImpl::MakeRoomForWrite(bool force) {
WritableFile* lfile = NULL; WritableFile* lfile = NULL;
s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile); s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
if (!s.ok()) { if (!s.ok()) {
// Avoid chewing through file number space in a tight loop.
versions_->ReuseFileNumber(new_log_number);
break; break;
} }
delete log_; delete log_;

View File

@ -94,7 +94,7 @@ class DBImpl : public DB {
void MaybeScheduleCompaction(); void MaybeScheduleCompaction();
static void BGWork(void* db); static void BGWork(void* db);
void BackgroundCall(); void BackgroundCall();
void BackgroundCompaction(); Status BackgroundCompaction();
void CleanupCompaction(CompactionState* compact); void CleanupCompaction(CompactionState* compact);
Status DoCompactionWork(CompactionState* compact); Status DoCompactionWork(CompactionState* compact);

View File

@ -56,12 +56,18 @@ class SpecialEnv : public EnvWrapper {
// Simulate no-space errors while this pointer is non-NULL. // Simulate no-space errors while this pointer is non-NULL.
port::AtomicPointer no_space_; port::AtomicPointer no_space_;
// Simulate non-writable file system while this pointer is non-NULL
port::AtomicPointer non_writable_;
bool count_random_reads_; bool count_random_reads_;
AtomicCounter random_read_counter_; AtomicCounter random_read_counter_;
AtomicCounter sleep_counter_;
explicit SpecialEnv(Env* base) : EnvWrapper(base) { explicit SpecialEnv(Env* base) : EnvWrapper(base) {
delay_sstable_sync_.Release_Store(NULL); delay_sstable_sync_.Release_Store(NULL);
no_space_.Release_Store(NULL); no_space_.Release_Store(NULL);
non_writable_.Release_Store(NULL);
count_random_reads_ = false; count_random_reads_ = false;
} }
@ -95,6 +101,10 @@ class SpecialEnv : public EnvWrapper {
} }
}; };
if (non_writable_.Acquire_Load() != NULL) {
return Status::IOError("simulated write error");
}
Status s = target()->NewWritableFile(f, r); Status s = target()->NewWritableFile(f, r);
if (s.ok()) { if (s.ok()) {
if (strstr(f.c_str(), ".sst") != NULL) { if (strstr(f.c_str(), ".sst") != NULL) {
@ -127,6 +137,11 @@ class SpecialEnv : public EnvWrapper {
} }
return s; return s;
} }
virtual void SleepForMicroseconds(int micros) {
sleep_counter_.Increment();
target()->SleepForMicroseconds(micros);
}
}; };
class DBTest { class DBTest {
@ -137,6 +152,7 @@ class DBTest {
enum OptionConfig { enum OptionConfig {
kDefault, kDefault,
kFilter, kFilter,
kUncompressed,
kEnd kEnd
}; };
int option_config_; int option_config_;
@ -167,10 +183,10 @@ class DBTest {
// Switch to a fresh database with the next option configuration to // Switch to a fresh database with the next option configuration to
// test. Return false if there are no more configurations to test. // test. Return false if there are no more configurations to test.
bool ChangeOptions() { bool ChangeOptions() {
if (option_config_ == kEnd) { option_config_++;
if (option_config_ >= kEnd) {
return false; return false;
} else { } else {
option_config_++;
DestroyAndReopen(); DestroyAndReopen();
return true; return true;
} }
@ -183,6 +199,9 @@ class DBTest {
case kFilter: case kFilter:
options.filter_policy = filter_policy_; options.filter_policy = filter_policy_;
break; break;
case kUncompressed:
options.compression = kNoCompression;
break;
default: default:
break; break;
} }
@ -552,13 +571,15 @@ TEST(DBTest, GetEncountersEmptyLevel) {
ASSERT_EQ(NumTableFilesAtLevel(1), 0); ASSERT_EQ(NumTableFilesAtLevel(1), 0);
ASSERT_EQ(NumTableFilesAtLevel(2), 1); ASSERT_EQ(NumTableFilesAtLevel(2), 1);
// Step 3: read until level 0 compaction disappears. // Step 3: read a bunch of times
int read_count = 0; for (int i = 0; i < 1000; i++) {
while (NumTableFilesAtLevel(0) > 0) {
ASSERT_LE(read_count, 10000) << "did not trigger level 0 compaction";
read_count++;
ASSERT_EQ("NOT_FOUND", Get("missing")); ASSERT_EQ("NOT_FOUND", Get("missing"));
} }
// Step 4: Wait for compaction to finish
env_->SleepForMicroseconds(1000000);
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
} while (ChangeOptions()); } while (ChangeOptions());
} }
@ -1432,13 +1453,37 @@ TEST(DBTest, NoSpace) {
Compact("a", "z"); Compact("a", "z");
const int num_files = CountFiles(); const int num_files = CountFiles();
env_->no_space_.Release_Store(env_); // Force out-of-space errors env_->no_space_.Release_Store(env_); // Force out-of-space errors
for (int i = 0; i < 10; i++) { env_->sleep_counter_.Reset();
for (int i = 0; i < 5; i++) {
for (int level = 0; level < config::kNumLevels-1; level++) { for (int level = 0; level < config::kNumLevels-1; level++) {
dbfull()->TEST_CompactRange(level, NULL, NULL); dbfull()->TEST_CompactRange(level, NULL, NULL);
} }
} }
env_->no_space_.Release_Store(NULL); env_->no_space_.Release_Store(NULL);
ASSERT_LT(CountFiles(), num_files + 5); ASSERT_LT(CountFiles(), num_files + 3);
// Check that compaction attempts slept after errors
ASSERT_GE(env_->sleep_counter_.Read(), 5);
}
TEST(DBTest, NonWritableFileSystem) {
Options options = CurrentOptions();
options.write_buffer_size = 1000;
options.env = env_;
Reopen(&options);
ASSERT_OK(Put("foo", "v1"));
env_->non_writable_.Release_Store(env_); // Force errors for new files
std::string big(100000, 'x');
int errors = 0;
for (int i = 0; i < 20; i++) {
fprintf(stderr, "iter %d; errors %d\n", i, errors);
if (!Put("foo", big).ok()) {
errors++;
env_->SleepForMicroseconds(100000);
}
}
ASSERT_GT(errors, 0);
env_->non_writable_.Release_Store(NULL);
} }
TEST(DBTest, FilesDeletedAfterCompaction) { TEST(DBTest, FilesDeletedAfterCompaction) {

View File

@ -132,7 +132,7 @@ bool SomeFileOverlapsRange(
const Comparator* ucmp = icmp.user_comparator(); const Comparator* ucmp = icmp.user_comparator();
if (!disjoint_sorted_files) { if (!disjoint_sorted_files) {
// Need to check against all files // Need to check against all files
for (int i = 0; i < files.size(); i++) { for (size_t i = 0; i < files.size(); i++) {
const FileMetaData* f = files[i]; const FileMetaData* f = files[i];
if (AfterFile(ucmp, smallest_user_key, f) || if (AfterFile(ucmp, smallest_user_key, f) ||
BeforeFile(ucmp, largest_user_key, f)) { BeforeFile(ucmp, largest_user_key, f)) {
@ -1297,7 +1297,7 @@ Compaction* VersionSet::CompactRange(
// Avoid compacting too much in one shot in case the range is large. // Avoid compacting too much in one shot in case the range is large.
const uint64_t limit = MaxFileSizeForLevel(level); const uint64_t limit = MaxFileSizeForLevel(level);
uint64_t total = 0; uint64_t total = 0;
for (int i = 0; i < inputs.size(); i++) { for (size_t i = 0; i < inputs.size(); i++) {
uint64_t s = inputs[i]->file_size; uint64_t s = inputs[i]->file_size;
total += s; total += s;
if (total >= limit) { if (total >= limit) {

View File

@ -173,6 +173,15 @@ class VersionSet {
// Allocate and return a new file number // Allocate and return a new file number
uint64_t NewFileNumber() { return next_file_number_++; } uint64_t NewFileNumber() { return next_file_number_++; }
// Arrange to reuse "file_number" unless a newer file number has
// already been allocated.
// REQUIRES: "file_number" was returned by a call to NewFileNumber().
void ReuseFileNumber(uint64_t file_number) {
if (next_file_number_ == file_number + 1) {
next_file_number_ = file_number;
}
}
// Return the number of Table files at the specified level. // Return the number of Table files at the specified level.
int NumLevelFiles(int level) const; int NumLevelFiles(int level) const;

View File

@ -75,6 +75,9 @@ static bool FLAGS_transaction = true;
// If true, we enable Write-Ahead Logging // If true, we enable Write-Ahead Logging
static bool FLAGS_WAL_enabled = true; static bool FLAGS_WAL_enabled = true;
// Use the db with the following name.
static const char* FLAGS_db = NULL;
inline inline
static void ExecErrorCheck(int status, char *err_msg) { static void ExecErrorCheck(int status, char *err_msg) {
if (status != SQLITE_OK) { if (status != SQLITE_OK) {
@ -317,11 +320,16 @@ class Benchmark {
bytes_(0), bytes_(0),
rand_(301) { rand_(301) {
std::vector<std::string> files; std::vector<std::string> files;
Env::Default()->GetChildren("/tmp", &files); std::string test_dir;
Env::Default()->GetTestDirectory(&test_dir);
Env::Default()->GetChildren(test_dir, &files);
if (!FLAGS_use_existing_db) { if (!FLAGS_use_existing_db) {
for (int i = 0; i < files.size(); i++) { for (int i = 0; i < files.size(); i++) {
if (Slice(files[i]).starts_with("dbbench_sqlite3")) { if (Slice(files[i]).starts_with("dbbench_sqlite3")) {
Env::Default()->DeleteFile("/tmp/" + files[i]); std::string file_name(test_dir);
file_name += "/";
file_name += files[i];
Env::Default()->DeleteFile(file_name.c_str());
} }
} }
} }
@ -415,7 +423,11 @@ class Benchmark {
db_num_++; db_num_++;
// Open database // Open database
snprintf(file_name, sizeof(file_name), "/tmp/dbbench_sqlite3-%d.db", std::string tmp_dir;
Env::Default()->GetTestDirectory(&tmp_dir);
snprintf(file_name, sizeof(file_name),
"%s/dbbench_sqlite3-%d.db",
tmp_dir.c_str(),
db_num_); db_num_);
status = sqlite3_open(file_name, &db_); status = sqlite3_open(file_name, &db_);
if (status) { if (status) {
@ -655,6 +667,7 @@ class Benchmark {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
std::string default_db_path;
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
double d; double d;
int n; int n;
@ -684,12 +697,21 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 && } else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) { (n == 0 || n == 1)) {
FLAGS_WAL_enabled = n; FLAGS_WAL_enabled = n;
} else if (strncmp(argv[i], "--db=", 5) == 0) {
FLAGS_db = argv[i] + 5;
} else { } else {
fprintf(stderr, "Invalid flag '%s'\n", argv[i]); fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
exit(1); exit(1);
} }
} }
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db == NULL) {
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench";
FLAGS_db = default_db_path.c_str();
}
leveldb::Benchmark benchmark; leveldb::Benchmark benchmark;
benchmark.Run(); benchmark.Run();
return 0; return 0;

View File

@ -68,6 +68,9 @@ static bool FLAGS_use_existing_db = false;
// is off. // is off.
static bool FLAGS_compression = true; static bool FLAGS_compression = true;
// Use the db with the following name.
static const char* FLAGS_db = NULL;
inline inline
static void DBSynchronize(kyotocabinet::TreeDB* db_) static void DBSynchronize(kyotocabinet::TreeDB* db_)
{ {
@ -292,11 +295,16 @@ class Benchmark {
bytes_(0), bytes_(0),
rand_(301) { rand_(301) {
std::vector<std::string> files; std::vector<std::string> files;
Env::Default()->GetChildren("/tmp", &files); std::string test_dir;
Env::Default()->GetTestDirectory(&test_dir);
Env::Default()->GetChildren(test_dir.c_str(), &files);
if (!FLAGS_use_existing_db) { if (!FLAGS_use_existing_db) {
for (int i = 0; i < files.size(); i++) { for (int i = 0; i < files.size(); i++) {
if (Slice(files[i]).starts_with("dbbench_polyDB")) { if (Slice(files[i]).starts_with("dbbench_polyDB")) {
Env::Default()->DeleteFile("/tmp/" + files[i]); std::string file_name(test_dir);
file_name += "/";
file_name += files[i];
Env::Default()->DeleteFile(file_name.c_str());
} }
} }
} }
@ -385,7 +393,11 @@ class Benchmark {
db_ = new kyotocabinet::TreeDB(); db_ = new kyotocabinet::TreeDB();
char file_name[100]; char file_name[100];
db_num_++; db_num_++;
snprintf(file_name, sizeof(file_name), "/tmp/dbbench_polyDB-%d.kct", std::string test_dir;
Env::Default()->GetTestDirectory(&test_dir);
snprintf(file_name, sizeof(file_name),
"%s/dbbench_polyDB-%d.kct",
test_dir.c_str(),
db_num_); db_num_);
// Create tuning options and open the database // Create tuning options and open the database
@ -470,6 +482,7 @@ class Benchmark {
} // namespace leveldb } // namespace leveldb
int main(int argc, char** argv) { int main(int argc, char** argv) {
std::string default_db_path;
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
double d; double d;
int n; int n;
@ -494,12 +507,21 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 && } else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) { (n == 0 || n == 1)) {
FLAGS_compression = (n == 1) ? true : false; FLAGS_compression = (n == 1) ? true : false;
} else if (strncmp(argv[i], "--db=", 5) == 0) {
FLAGS_db = argv[i] + 5;
} else { } else {
fprintf(stderr, "Invalid flag '%s'\n", argv[i]); fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
exit(1); exit(1);
} }
} }
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db == NULL) {
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench";
FLAGS_db = default_db_path.c_str();
}
leveldb::Benchmark benchmark; leveldb::Benchmark benchmark;
benchmark.Run(); benchmark.Run();
return 0; return 0;

View File

@ -73,13 +73,21 @@ inline void MemoryBarrier() {
} }
#define LEVELDB_HAVE_MEMORY_BARRIER #define LEVELDB_HAVE_MEMORY_BARRIER
// ARM // ARM Linux
#elif defined(ARCH_CPU_ARM_FAMILY) #elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__)
typedef void (*LinuxKernelMemoryBarrierFunc)(void); typedef void (*LinuxKernelMemoryBarrierFunc)(void);
LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier __attribute__((weak)) = // The Linux ARM kernel provides a highly optimized device-specific memory
(LinuxKernelMemoryBarrierFunc) 0xffff0fa0; // barrier function at a fixed memory address that is mapped in every
// user-level process.
//
// This beats using CPU-specific instructions which are, on single-core
// devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more
// than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking
// shows that the extra function call cost is completely negligible on
// multi-core devices.
//
inline void MemoryBarrier() { inline void MemoryBarrier() {
pLinuxKernelMemoryBarrier(); (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)();
} }
#define LEVELDB_HAVE_MEMORY_BARRIER #define LEVELDB_HAVE_MEMORY_BARRIER

View File

@ -14,8 +14,6 @@
# include "port/port_posix.h" # include "port/port_posix.h"
#elif defined(LEVELDB_PLATFORM_CHROMIUM) #elif defined(LEVELDB_PLATFORM_CHROMIUM)
# include "port/port_chromium.h" # include "port/port_chromium.h"
#elif defined(LEVELDB_PLATFORM_ANDROID)
# include "port/port_android.h"
#endif #endif
#endif // STORAGE_LEVELDB_PORT_PORT_H_ #endif // STORAGE_LEVELDB_PORT_PORT_H_

View File

@ -60,6 +60,16 @@ class CondVar {
void SignallAll(); void SignallAll();
}; };
// Thread-safe initialization.
// Used as follows:
// static port::OnceType init_control = LEVELDB_ONCE_INIT;
// static void Initializer() { ... do something ...; }
// ...
// port::InitOnce(&init_control, &Initializer);
typedef intptr_t OnceType;
#define LEVELDB_ONCE_INIT 0
extern void InitOnce(port::OnceType*, void (*initializer)());
// A type that holds a pointer that can be read or written atomically // A type that holds a pointer that can be read or written atomically
// (i.e., without word-tearing.) // (i.e., without word-tearing.)
class AtomicPointer { class AtomicPointer {

View File

@ -46,5 +46,9 @@ void CondVar::SignalAll() {
PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); PthreadCall("broadcast", pthread_cond_broadcast(&cv_));
} }
void InitOnce(OnceType* once, void (*initializer)()) {
PthreadCall("once", pthread_once(once, initializer));
}
} // namespace port } // namespace port
} // namespace leveldb } // namespace leveldb

View File

@ -7,17 +7,22 @@
#ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_ #ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_
#define STORAGE_LEVELDB_PORT_PORT_POSIX_H_ #define STORAGE_LEVELDB_PORT_PORT_POSIX_H_
#undef PLATFORM_IS_LITTLE_ENDIAN
#if defined(OS_MACOSX) #if defined(OS_MACOSX)
#include <machine/endian.h> #include <machine/endian.h>
#if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER)
#define PLATFORM_IS_LITTLE_ENDIAN \
(__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN)
#endif
#elif defined(OS_SOLARIS) #elif defined(OS_SOLARIS)
#include <sys/isa_defs.h> #include <sys/isa_defs.h>
#ifdef _LITTLE_ENDIAN #ifdef _LITTLE_ENDIAN
#define LITTLE_ENDIAN #define PLATFORM_IS_LITTLE_ENDIAN true
#else #else
#define BIG_ENDIAN #define PLATFORM_IS_LITTLE_ENDIAN false
#endif #endif
#elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) ||\ #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) ||\
defined(OS_DRAGONFLYBSD) defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID)
#include <sys/types.h> #include <sys/types.h>
#include <sys/endian.h> #include <sys/endian.h>
#else #else
@ -31,14 +36,13 @@
#include <string> #include <string>
#include "port/atomic_pointer.h" #include "port/atomic_pointer.h"
#ifdef LITTLE_ENDIAN #ifndef PLATFORM_IS_LITTLE_ENDIAN
#define IS_LITTLE_ENDIAN true #define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN)
#else
#define IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN)
#endif #endif
#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\
defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\
defined(OS_ANDROID)
// Use fread/fwrite/fflush on platforms without _unlocked variants // Use fread/fwrite/fflush on platforms without _unlocked variants
#define fread_unlocked fread #define fread_unlocked fread
#define fwrite_unlocked fwrite #define fwrite_unlocked fwrite
@ -51,10 +55,17 @@
#define fdatasync fsync #define fdatasync fsync
#endif #endif
#if defined(OS_ANDROID) && __ANDROID_API__ < 9
// fdatasync() was only introduced in API level 9 on Android. Use fsync()
// when targetting older platforms.
#define fdatasync fsync
#endif
namespace leveldb { namespace leveldb {
namespace port { namespace port {
static const bool kLittleEndian = IS_LITTLE_ENDIAN; static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN;
#undef PLATFORM_IS_LITTLE_ENDIAN
class CondVar; class CondVar;
@ -88,6 +99,10 @@ class CondVar {
Mutex* mu_; Mutex* mu_;
}; };
typedef pthread_once_t OnceType;
#define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT
extern void InitOnce(OnceType* once, void (*initializer)());
inline bool Snappy_Compress(const char* input, size_t length, inline bool Snappy_Compress(const char* input, size_t length,
::std::string* output) { ::std::string* output) {
#ifdef SNAPPY #ifdef SNAPPY

View File

@ -51,6 +51,29 @@ TEST(Coding, Fixed64) {
} }
} }
// Test that encoding routines generate little-endian encodings
TEST(Coding, EncodingOutput) {
std::string dst;
PutFixed32(&dst, 0x04030201);
ASSERT_EQ(4, dst.size());
ASSERT_EQ(0x01, static_cast<int>(dst[0]));
ASSERT_EQ(0x02, static_cast<int>(dst[1]));
ASSERT_EQ(0x03, static_cast<int>(dst[2]));
ASSERT_EQ(0x04, static_cast<int>(dst[3]));
dst.clear();
PutFixed64(&dst, 0x0807060504030201ull);
ASSERT_EQ(8, dst.size());
ASSERT_EQ(0x01, static_cast<int>(dst[0]));
ASSERT_EQ(0x02, static_cast<int>(dst[1]));
ASSERT_EQ(0x03, static_cast<int>(dst[2]));
ASSERT_EQ(0x04, static_cast<int>(dst[3]));
ASSERT_EQ(0x05, static_cast<int>(dst[4]));
ASSERT_EQ(0x06, static_cast<int>(dst[5]));
ASSERT_EQ(0x07, static_cast<int>(dst[6]));
ASSERT_EQ(0x08, static_cast<int>(dst[7]));
}
TEST(Coding, Varint32) { TEST(Coding, Varint32) {
std::string s; std::string s;
for (uint32_t i = 0; i < (32 * 32); i++) { for (uint32_t i = 0; i < (32 * 32); i++) {

View File

@ -6,6 +6,7 @@
#include <stdint.h> #include <stdint.h>
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
#include "leveldb/slice.h" #include "leveldb/slice.h"
#include "port/port.h"
#include "util/logging.h" #include "util/logging.h"
namespace leveldb { namespace leveldb {
@ -65,11 +66,15 @@ class BytewiseComparatorImpl : public Comparator {
}; };
} // namespace } // namespace
// Intentionally not destroyed to prevent destructor racing static port::OnceType once = LEVELDB_ONCE_INIT;
// with background threads. static const Comparator* bytewise;
static const Comparator* bytewise = new BytewiseComparatorImpl;
static void InitModule() {
bytewise = new BytewiseComparatorImpl;
}
const Comparator* BytewiseComparator() { const Comparator* BytewiseComparator() {
port::InitOnce(&once, InitModule);
return bytewise; return bytewise;
} }