tqcq/leveldb
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Upstream changes.

Browse Source 
git-svn-id: https://leveldb.googlecode.com/svn/trunk@16 62dab493-f737-651d-591e-8d6aee1b9529
This commit is contained in:
jorlow@chromium.org 2011-03-28 20:43:44 +00:00
parent e11bdf1935
commit e2da744e12
17 changed files with 67 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
db/builder.cc
View File

@ -74,7 +74,9 @@ Status BuildTable(const std::string& dbname,
if (s.ok()) {
// Verify that the table is usable
Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number);
Iterator* it = table_cache->NewIterator(ReadOptions(),
meta->number,
meta->file_size);
s = it->status();
delete it;
}

4
db/db_bench.cc
View File

@ -354,7 +354,7 @@ class Benchmark {
private:
void Crc32c(int size, const char* label) {
// Checksum about 500MB of data total
string data(size, 'x');
std::string data(size, 'x');
int64_t bytes = 0;
uint32_t crc = 0;
while (bytes < 500 * 1048576) {
@ -371,7 +371,7 @@ class Benchmark {
void SHA1(int size, const char* label) {
// SHA1 about 100MB of data total
string data(size, 'x');
std::string data(size, 'x');
int64_t bytes = 0;
char sha1[20];
while (bytes < 100 * 1048576) {

4
db/db_impl.cc
View File

@ -642,7 +642,9 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
if (s.ok() && current_entries > 0) {
// Verify that the table is usable
Iterator* iter = table_cache_->NewIterator(ReadOptions(),output_number);
Iterator* iter = table_cache_->NewIterator(ReadOptions(),
output_number,
current_bytes);
s = iter->status();
delete iter;
if (s.ok()) {

5
db/db_iter.cc
View File

@ -340,8 +340,11 @@ void DBIter::ReadIndirectValue(Slice ref) const {
std::string fname = LargeValueFileName(*dbname_, large_ref);
RandomAccessFile* file;
Status s = env_->NewRandomAccessFile(fname, &file);
uint64_t file_size = 0;
if (s.ok()) {
s = env_->GetFileSize(fname, &file_size);
}
if (s.ok()) {
uint64_t file_size = file->Size();
uint64_t value_size = large_ref.ValueSize();
large_->value.resize(value_size);
Slice result;

6
db/log_reader.cc
View File

@ -105,7 +105,7 @@ void Reader::ReportDrop(size_t bytes, const char* reason) {
unsigned int Reader::ReadPhysicalRecord(Slice* result) {
while (true) {
if (buffer_.size() <= kHeaderSize) {
if (buffer_.size() < kHeaderSize) {
if (!eof_) {
// Last read was a full read, so this is a trailer to skip
buffer_.clear();
@ -124,12 +124,10 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
} else if (buffer_.size() == 0) {
// End of file
return kEof;
} else if (buffer_.size() < kHeaderSize) {
} else {
ReportDrop(buffer_.size(), "truncated record at end of file");
buffer_.clear();
return kEof;
} else {
// We have a trailing zero-length record. Fall through and check it.
}
}

11
db/log_writer.cc
View File

@ -35,18 +35,19 @@ Status Writer::AddRecord(const Slice& slice) {
do {
const int leftover = kBlockSize - block_offset_;
assert(leftover >= 0);
if (leftover <= kHeaderSize) {
if (leftover < kHeaderSize) {
// Switch to a new block
if (leftover > 0) {
// Fill the trailer
dest_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00", leftover));
// Fill the trailer (literal below relies on kHeaderSize being 7)
assert(kHeaderSize == 7);
dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
}
block_offset_ = 0;
}
// Invariant: we never leave <= kHeaderSize bytes in a block.
// Invariant: we never leave < kHeaderSize bytes in a block.
const int avail = kBlockSize - block_offset_ - kHeaderSize;
assert(avail > 0);
assert(avail >= 0);
const size_t fragment_length = (left < avail) ? left : avail;

2
db/repair.cc
View File

@ -261,7 +261,7 @@ class Repairer {
Status status = env_->GetFileSize(fname, &t->meta.file_size);
if (status.ok()) {
Iterator* iter = table_cache_->NewIterator(
ReadOptions(), t->meta.number);
ReadOptions(), t->meta.number, t->meta.file_size);
bool empty = true;
ParsedInternalKey parsed;
t->max_sequence = 0;

3
db/table_cache.cc
View File

@ -44,6 +44,7 @@ TableCache::~TableCache() {
Iterator* TableCache::NewIterator(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
Table** tableptr) {
if (tableptr != NULL) {
*tableptr = NULL;
@ -59,7 +60,7 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
Table* table = NULL;
Status s = env_->NewRandomAccessFile(fname, &file);
if (s.ok()) {
s = Table::Open(*options_, file, &table);
s = Table::Open(*options_, file, file_size, &table);
}
if (!s.ok()) {

15
db/table_cache.h
View File

@ -23,15 +23,16 @@ class TableCache {
TableCache(const std::string& dbname, const Options* options, int entries);
~TableCache();
// Get an iterator for the specified file number and return it. If
// "tableptr" is non-NULL, also sets "*tableptr" to point to the
// Table object underlying the returned iterator, or NULL if no
// Table object underlies the returned iterator. The returned
// "*tableptr" object is owned by the cache and should not be
// deleted, and is valid for as long as the returned iterator is
// live.
// Return an iterator for the specified file number (the corresponding
// file length must be exactly "file_size" bytes). If "tableptr" is
// non-NULL, also sets "*tableptr" to point to the Table object
// underlying the returned iterator, or NULL if no Table object underlies
// the returned iterator. The returned "*tableptr" object is owned by
// the cache and should not be deleted, and is valid for as long as the
// returned iterator is live.
Iterator* NewIterator(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
Table** tableptr = NULL);
// Evict any entry for the specified file number

22
db/version_set.cc
View File

@ -75,8 +75,8 @@ Version::~Version() {
// An internal iterator. For a given version/level pair, yields
// information about the files in the level. For a given entry, key()
// is the largest key that occurs in the file, and value() is an
// 8-byte value containing the file number of the file, encoding using
// EncodeFixed64.
// 16-byte value containing the file number and file size, both
// encoded using EncodeFixed64.
class Version::LevelFileNumIterator : public Iterator {
public:
LevelFileNumIterator(const Version* version,
@ -129,6 +129,7 @@ class Version::LevelFileNumIterator : public Iterator {
Slice value() const {
assert(Valid());
EncodeFixed64(value_buf_, (*flist_)[index_]->number);
EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
return Slice(value_buf_, sizeof(value_buf_));
}
virtual Status status() const { return Status::OK(); }
@ -137,18 +138,21 @@ class Version::LevelFileNumIterator : public Iterator {
const std::vector<FileMetaData*>* const flist_;
int index_;
mutable char value_buf_[8]; // Used for encoding the file number for value()
// Backing store for value(). Holds the file number and size.
mutable char value_buf_[16];
};
static Iterator* GetFileIterator(void* arg,
const ReadOptions& options,
const Slice& file_value) {
TableCache* cache = reinterpret_cast<TableCache*>(arg);
if (file_value.size() != 8) {
if (file_value.size() != 16) {
return NewErrorIterator(
Status::Corruption("FileReader invoked with unexpected value"));
} else {
return cache->NewIterator(options, DecodeFixed64(file_value.data()));
return cache->NewIterator(options,
DecodeFixed64(file_value.data()),
DecodeFixed64(file_value.data() + 8));
}
}
@ -164,7 +168,8 @@ void Version::AddIterators(const ReadOptions& options,
// Merge all level zero files together since they may overlap
for (int i = 0; i < files_[0].size(); i++) {
iters->push_back(
vset_->table_cache_->NewIterator(options, files_[0][i]->number));
vset_->table_cache_->NewIterator(
options, files_[0][i]->number, files_[0][i]->file_size));
}
// For levels > 0, we can use a concatenating iterator that sequentially
@ -650,7 +655,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
// approximate offset of "ikey" within the table.
Table* tableptr;
Iterator* iter = table_cache_->NewIterator(
ReadOptions(), files[i]->number, &tableptr);
ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
if (tableptr != NULL) {
result += tableptr->ApproximateOffsetOf(ikey.Encode());
}
@ -855,7 +860,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
if (c->level() + which == 0) {
const std::vector<FileMetaData*>& files = c->inputs_[which];
for (int i = 0; i < files.size(); i++) {
list[num++] = table_cache_->NewIterator(options, files[i]->number);
list[num++] = table_cache_->NewIterator(
options, files[i]->number, files[i]->file_size);
}
} else {
// Create concatenating iterator for the files from this level

15
doc/log_format.txt
View File

@ -9,12 +9,15 @@ Each block consists of a sequence of records:
type: uint8 // One of FULL, FIRST, MIDDLE, LAST
data: uint8[length]
A record never starts within the last seven bytes of a block. Any
leftover bytes here form the trailer, which must consist entirely of
zero bytes and must be skipped by readers. In particular, even if
there are exactly seven bytes left in the block, and a zero-length
user record is added (which will fit in these seven bytes), the writer
must skip these trailer bytes and add the record to the next block.
A record never starts within the last six bytes of a block (since it
won't fit). Any leftover bytes here form the trailer, which must
consist entirely of zero bytes and must be skipped by readers.
Aside: if exactly seven bytes are left in the current block, and a new
non-zero length record is added, the writer must emit a FIRST record
(which contains zero bytes of user data) to fill up the trailing seven
bytes of the block and then emit all of the user data in subsequent
blocks.
More types may be added in the future. Some Readers may skip record
types they do not understand, others may report that some data was

3
include/env.h
View File

@ -168,9 +168,6 @@ class RandomAccessFile {
RandomAccessFile() { }
virtual ~RandomAccessFile();
// Return the length of this file in bytes.
virtual uint64_t Size() const = 0;
// Read up to "n" bytes from the file starting at "offset".
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
// to the data that was read (including if fewer than "n" bytes were

6
include/table.h
View File

@ -20,8 +20,9 @@ struct ReadOptions;
// immutable and persistent.
class Table {
public:
// Attempt to open the table that is stored in "file", and read the
// metadata entries necessary to allow retrieving data from the table.
// Attempt to open the table that is stored in bytes [0..file_size)
// of "file", and read the metadata entries necessary to allow
// retrieving data from the table.
//
// If successful, returns ok and sets "*table" to the newly opened
// table. The client should delete "*table" when no longer needed.
@ -33,6 +34,7 @@ class Table {
// *file must remain live while this Table is in use.
static Status Open(const Options& options,
RandomAccessFile* file,
uint64_t file_size,
Table** table);
~Table();

2
table/table.cc
View File

@ -29,9 +29,9 @@ struct Table::Rep {
Status Table::Open(const Options& options,
RandomAccessFile* file,
uint64_t size,
Table** table) {
*table = NULL;
const uint64_t size = file->Size();
if (size < Footer::kEncodedLength) {
return Status::InvalidArgument("file is too short to be an sstable");
}

4
table/table_test.cc
View File

@ -110,7 +110,7 @@ class StringSource: public RandomAccessFile {
virtual ~StringSource() { }
virtual uint64_t Size() const { return contents_.size(); }
uint64_t Size() const { return contents_.size(); }
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const {
@ -246,7 +246,7 @@ class TableConstructor: public Constructor {
source_ = new StringSource(sink.contents());
Options table_options;
table_options.comparator = options.comparator;
return Table::Open(table_options, source_, &table_);
return Table::Open(table_options, source_, sink.contents().size(), &table_);
}
virtual size_t NumBytes() const { return source_->Size(); }

16
util/env_chromium.cc
View File

@ -144,17 +144,13 @@ class ChromiumSequentialFile: public SequentialFile {
class ChromiumRandomAccessFile: public RandomAccessFile {
private:
std::string filename_;
uint64_t size_;
::base::PlatformFile file_;
public:
ChromiumRandomAccessFile(const std::string& fname, uint64_t size,
::base::PlatformFile file)
: filename_(fname), size_(size), file_(file) { }
ChromiumRandomAccessFile(const std::string& fname, ::base::PlatformFile file)
: filename_(fname), file_(file) { }
virtual ~ChromiumRandomAccessFile() { ::base::ClosePlatformFile(file_); }
virtual uint64_t Size() const { return size_; }
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const {
Status s;
@ -256,13 +252,7 @@ class ChromiumEnv : public Env {
*result = NULL;
return Status::IOError(fname, PlatformFileErrorString(error_code));
}
::base::PlatformFileInfo info;
if (!::base::GetPlatformFileInfo(file, &info)) {
*result = NULL;
::base::ClosePlatformFile(file);
return Status::IOError(fname, PlatformFileErrorString(error_code));
}
*result = new ChromiumRandomAccessFile(fname, info.size, file);
*result = new ChromiumRandomAccessFile(fname, file);
return Status::OK();
}

16
util/env_posix.cc
View File

@ -57,16 +57,13 @@ class PosixSequentialFile: public SequentialFile {
class PosixRandomAccessFile: public RandomAccessFile {
private:
std::string filename_;
uint64_t size_;
int fd_;
public:
PosixRandomAccessFile(const std::string& fname, uint64_t size, int fd)
: filename_(fname), size_(size), fd_(fd) { }
PosixRandomAccessFile(const std::string& fname, int fd)
: filename_(fname), fd_(fd) { }
virtual ~PosixRandomAccessFile() { close(fd_); }
virtual uint64_t Size() const { return size_; }
virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const {
Status s;
@ -286,14 +283,7 @@ class PosixEnv : public Env {
*result = NULL;
return Status::IOError(fname, strerror(errno));
}
struct stat sbuf;
if (fstat(fd, &sbuf) != 0) {
*result = NULL;
Status s = Status::IOError(fname, strerror(errno));
close(fd);
return s;
}
*result = new PosixRandomAccessFile(fname, sbuf.st_size, fd);
*result = new PosixRandomAccessFile(fname, fd);
return Status::OK();
}