Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storage/engine: expose FailOnMoreRecent MVCC{Get/Scan}Option #44473

Merged
merged 3 commits into from
Feb 1, 2020
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 39 additions & 13 deletions c-deps/libroach/batch.cc
Original file line number Diff line number Diff line change
Expand Up @@ -587,14 +587,25 @@ DBStatus DBBatch::EnvDeleteDirAndFiles(DBSlice dir) { return FmtStatus("unsuppor

DBStatus DBBatch::EnvLinkFile(DBSlice oldname, DBSlice newname) { return FmtStatus("unsupported"); }

DBStatus DBBatch::EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file) { return FmtStatus("unsupported"); }
DBStatus DBBatch::EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n) { return FmtStatus("unsupported"); }
DBStatus DBBatch::EnvCloseReadableFile(rocksdb::RandomAccessFile* file) { return FmtStatus("unsupported"); }
DBStatus DBBatch::EnvOpenDirectory(DBSlice path, rocksdb::Directory** file) { return FmtStatus("unsupported"); }
DBStatus DBBatch::EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file) {
return FmtStatus("unsupported");
}
DBStatus DBBatch::EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset,
int* n) {
return FmtStatus("unsupported");
}
DBStatus DBBatch::EnvCloseReadableFile(rocksdb::RandomAccessFile* file) {
return FmtStatus("unsupported");
}
DBStatus DBBatch::EnvOpenDirectory(DBSlice path, rocksdb::Directory** file) {
return FmtStatus("unsupported");
}
DBStatus DBBatch::EnvSyncDirectory(rocksdb::Directory* file) { return FmtStatus("unsupported"); }
DBStatus DBBatch::EnvCloseDirectory(rocksdb::Directory* file) { return FmtStatus("unsupported"); }
DBStatus DBBatch::EnvRenameFile(DBSlice oldname, DBSlice newname) { return FmtStatus("unsupported"); }

DBStatus DBBatch::EnvRenameFile(DBSlice oldname, DBSlice newname) {
return FmtStatus("unsupported");
}

DBWriteOnlyBatch::DBWriteOnlyBatch(DBEngine* db) : DBEngine(db->rep, db->iters), updates(0) {}

DBWriteOnlyBatch::~DBWriteOnlyBatch() {}
Expand Down Expand Up @@ -706,13 +717,28 @@ DBStatus DBWriteOnlyBatch::EnvLinkFile(DBSlice oldname, DBSlice newname) {
return FmtStatus("unsupported");
}

DBStatus DBWriteOnlyBatch::EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvCloseReadableFile(rocksdb::RandomAccessFile* file) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvOpenDirectory(DBSlice path, rocksdb::Directory** file) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvSyncDirectory(rocksdb::Directory* file) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvCloseDirectory(rocksdb::Directory* file) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvRenameFile(DBSlice oldname, DBSlice newname) { return FmtStatus("unsupported"); }
DBStatus DBWriteOnlyBatch::EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file) {
return FmtStatus("unsupported");
}
DBStatus DBWriteOnlyBatch::EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer,
int64_t offset, int* n) {
return FmtStatus("unsupported");
}
DBStatus DBWriteOnlyBatch::EnvCloseReadableFile(rocksdb::RandomAccessFile* file) {
return FmtStatus("unsupported");
}
DBStatus DBWriteOnlyBatch::EnvOpenDirectory(DBSlice path, rocksdb::Directory** file) {
return FmtStatus("unsupported");
}
DBStatus DBWriteOnlyBatch::EnvSyncDirectory(rocksdb::Directory* file) {
return FmtStatus("unsupported");
}
DBStatus DBWriteOnlyBatch::EnvCloseDirectory(rocksdb::Directory* file) {
return FmtStatus("unsupported");
}
DBStatus DBWriteOnlyBatch::EnvRenameFile(DBSlice oldname, DBSlice newname) {
return FmtStatus("unsupported");
}

rocksdb::WriteBatch::Handler* GetDBBatchInserter(::rocksdb::WriteBatchBase* batch) {
return new DBBatchInserter(batch);
Expand Down
6 changes: 4 additions & 2 deletions c-deps/libroach/batch.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ struct DBBatch : public DBEngine {
virtual DBStatus EnvDeleteDirAndFiles(DBSlice dir);
virtual DBStatus EnvLinkFile(DBSlice oldname, DBSlice newname);
virtual DBStatus EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file);
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n);
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset,
int* n);
virtual DBStatus EnvCloseReadableFile(rocksdb::RandomAccessFile* file);
virtual DBStatus EnvOpenDirectory(DBSlice path, rocksdb::Directory** file);
virtual DBStatus EnvSyncDirectory(rocksdb::Directory* file);
Expand Down Expand Up @@ -90,7 +91,8 @@ struct DBWriteOnlyBatch : public DBEngine {
virtual DBStatus EnvDeleteDirAndFiles(DBSlice dir);
virtual DBStatus EnvLinkFile(DBSlice oldname, DBSlice newname);
virtual DBStatus EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file);
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n);
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset,
int* n);
virtual DBStatus EnvCloseReadableFile(rocksdb::RandomAccessFile* file);
virtual DBStatus EnvOpenDirectory(DBSlice path, rocksdb::Directory** file);
virtual DBStatus EnvSyncDirectory(rocksdb::Directory* file);
Expand Down
6 changes: 2 additions & 4 deletions c-deps/libroach/ccl/db_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,7 @@ TEST_F(CCLTest, ReadOnly) {
free(ro_value.data);
// Try to write it again.
auto ret = DBPut(db, ToDBKey("foo"), ToDBSlice("foo's value"));
EXPECT_EQ(ToString(ret),
"Not implemented: Not supported operation in read only mode.");
EXPECT_EQ(ToString(ret), "Not implemented: Not supported operation in read only mode.");
free(ret.data);

DBClose(db);
Expand All @@ -192,8 +191,7 @@ TEST_F(CCLTest, ReadOnly) {
free(ro_value.data);
// Try to write it again.
auto ret = DBPut(db, ToDBKey("foo"), ToDBSlice("foo's value"));
EXPECT_EQ(ToString(ret),
"Not implemented: Not supported operation in read only mode.");
EXPECT_EQ(ToString(ret), "Not implemented: Not supported operation in read only mode.");
free(ret.data);

DBClose(db);
Expand Down
2 changes: 1 addition & 1 deletion c-deps/libroach/chunked_buffer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ void chunkedBuffer::put(const char* data, int len, int next_size_hint) {
data += avail;
len -= avail;

const int max_size = 128 << 20; // 128 MB
const int max_size = 128 << 20; // 128 MB
size_t new_size = bufs_.empty() ? 16 : bufs_.back().len * 2;
for (; new_size < len + next_size_hint && new_size < max_size; new_size *= 2) {
}
Expand Down
8 changes: 4 additions & 4 deletions c-deps/libroach/chunked_buffer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
// writing in pieces that are smaller than the maximum chunk size (128
// MB). See #32896.
TEST(ChunkedBuffer, PutSmall) {
const std::string data(1 << 20, '.'); // 1 MB
const int64_t total = 3LL << 30; // 3 GB
const std::string data(1 << 20, '.'); // 1 MB
const int64_t total = 3LL << 30; // 3 GB
cockroach::chunkedBuffer buf;
for (int64_t sum = 0; sum < total; sum += data.size()) {
buf.Put(data, data);
Expand All @@ -27,8 +27,8 @@ TEST(ChunkedBuffer, PutSmall) {
// writing in pieces that are larger than the maximum chunk size (128
// MB). See #32896.
TEST(ChunkedBuffer, PutLarge) {
const std::string data(256 << 20, '.'); // 256 MB
const int64_t total = 3LL << 30; // 3 GB
const std::string data(256 << 20, '.'); // 256 MB
const int64_t total = 3LL << 30; // 3 GB
cockroach::chunkedBuffer buf;
for (int64_t sum = 0; sum < total; sum += data.size()) {
buf.Put(data, data);
Expand Down
13 changes: 7 additions & 6 deletions c-deps/libroach/comparator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,13 @@ bool DBComparator::Equal(const rocksdb::Slice& a, const rocksdb::Slice& b) const

namespace {

void ShrinkSlice(rocksdb::Slice* a, size_t size) {
a->remove_suffix(a->size() - size);
}
void ShrinkSlice(rocksdb::Slice* a, size_t size) { a->remove_suffix(a->size() - size); }

int SharedPrefixLen(const rocksdb::Slice& a, const rocksdb::Slice& b) {
auto n = std::min(a.size(), b.size());
int i = 0;
for (; i < n && a[i] == b[i]; ++i) {}
for (; i < n && a[i] == b[i]; ++i) {
}
return i;
}

Expand All @@ -68,7 +67,8 @@ bool FindSeparator(rocksdb::Slice* a, std::string* a_backing, const rocksdb::Sli
// So b is smaller than a.
return false;
}
if ((prefix < b.size() - 1) || static_cast<unsigned char>((*a)[prefix]) + 1 < static_cast<unsigned char>(b[prefix])) {
if ((prefix < b.size() - 1) ||
static_cast<unsigned char>((*a)[prefix]) + 1 < static_cast<unsigned char>(b[prefix])) {
// a and b do not have consecutive characters at prefix.
(*a_backing)[prefix]++;
ShrinkSlice(a, prefix + 1);
Expand Down Expand Up @@ -97,7 +97,8 @@ void DBComparator::FindShortestSeparator(std::string* start, const rocksdb::Slic
return;
}
auto found = FindSeparator(&key_s, start, key_l);
if (!found) return;
if (!found)
return;
start->resize(key_s.size() + 1);
(*start)[key_s.size()] = 0x00;
}
Expand Down
86 changes: 43 additions & 43 deletions c-deps/libroach/comparator_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -40,39 +40,39 @@ DBKey makeKey(const char* s, int64_t wall_time = 0, int32_t logical = 0) {
TEST(Libroach, Comparator) {
DBComparator comp;
std::vector<SeparatorTestCase> sepCases = {
// Many cases here are adapted from a Pebble unit test.
// Many cases here are adapted from a Pebble unit test.

// Non-empty b values.
{makeKey("black"), makeKey("blue"), makeKey("blb")},
{makeKey(""), makeKey("2"), makeKey("")},
{makeKey("1"), makeKey("2"), makeKey("1")},
{makeKey("1"), makeKey("29"), makeKey("2")},
{makeKey("13"), makeKey("19"), makeKey("14")},
{makeKey("13"), makeKey("99"), makeKey("2")},
{makeKey("135"), makeKey("19"), makeKey("14")},
{makeKey("1357"), makeKey("19"), makeKey("14")},
{makeKey("1357"), makeKey("2"), makeKey("14")},
{makeKey("13\xff"), makeKey("14"), makeKey("13\xff")},
{makeKey("13\xff"), makeKey("19"), makeKey("14")},
{makeKey("1\xff\xff"), makeKey("19"), makeKey("1\xff\xff")},
{makeKey("1\xff\xff"), makeKey("2"), makeKey("1\xff\xff")},
{makeKey("1\xff\xff"), makeKey("9"), makeKey("2")},
{makeKey("1\xfd\xff"), makeKey("1\xff"), makeKey("1\xfe")},
{makeKey("1\xff\xff", 20, 3), makeKey("9"), makeKey("2")},
{makeKey("1\xff\xff", 20, 3), makeKey("19"), makeKey("1\xff\xff", 20, 3)},
// Non-empty b values.
{makeKey("black"), makeKey("blue"), makeKey("blb")},
{makeKey(""), makeKey("2"), makeKey("")},
{makeKey("1"), makeKey("2"), makeKey("1")},
{makeKey("1"), makeKey("29"), makeKey("2")},
{makeKey("13"), makeKey("19"), makeKey("14")},
{makeKey("13"), makeKey("99"), makeKey("2")},
{makeKey("135"), makeKey("19"), makeKey("14")},
{makeKey("1357"), makeKey("19"), makeKey("14")},
{makeKey("1357"), makeKey("2"), makeKey("14")},
{makeKey("13\xff"), makeKey("14"), makeKey("13\xff")},
{makeKey("13\xff"), makeKey("19"), makeKey("14")},
{makeKey("1\xff\xff"), makeKey("19"), makeKey("1\xff\xff")},
{makeKey("1\xff\xff"), makeKey("2"), makeKey("1\xff\xff")},
{makeKey("1\xff\xff"), makeKey("9"), makeKey("2")},
{makeKey("1\xfd\xff"), makeKey("1\xff"), makeKey("1\xfe")},
{makeKey("1\xff\xff", 20, 3), makeKey("9"), makeKey("2")},
{makeKey("1\xff\xff", 20, 3), makeKey("19"), makeKey("1\xff\xff", 20, 3)},

// Empty b values
{makeKey(""), makeKey(""), makeKey("")},
{makeKey("green"), makeKey(""), makeKey("green")},
{makeKey("1"), makeKey(""), makeKey("1")},
{makeKey("11\xff"), makeKey(""), makeKey("11\xff")},
{makeKey("1\xff"), makeKey(""), makeKey("1\xff")},
{makeKey("1\xff\xff"), makeKey(""), makeKey("1\xff\xff")},
{makeKey("\xff"), makeKey(""), makeKey("\xff")},
{makeKey("\xff\xff"), makeKey(""), makeKey("\xff\xff")},
// Empty b values
{makeKey(""), makeKey(""), makeKey("")},
{makeKey("green"), makeKey(""), makeKey("green")},
{makeKey("1"), makeKey(""), makeKey("1")},
{makeKey("11\xff"), makeKey(""), makeKey("11\xff")},
{makeKey("1\xff"), makeKey(""), makeKey("1\xff")},
{makeKey("1\xff\xff"), makeKey(""), makeKey("1\xff\xff")},
{makeKey("\xff"), makeKey(""), makeKey("\xff")},
{makeKey("\xff\xff"), makeKey(""), makeKey("\xff\xff")},
};
for (const auto& c: sepCases) {

for (const auto& c : sepCases) {
auto a_str = EncodeKey(c.a);
auto b_str = EncodeKey(c.b);
std::printf("a_str: %s, b_str: %s\n", a_str.c_str(), b_str.c_str());
Expand All @@ -81,21 +81,21 @@ TEST(Libroach, Comparator) {
}

std::vector<SuccessorTestCase> succCases = {
{makeKey("black"), makeKey("c")},
{makeKey("green"), makeKey("h")},
{makeKey(""), makeKey("")},
{makeKey("13"), makeKey("2")},
{makeKey("135"), makeKey("2")},
{makeKey("13\xff"), makeKey("2")},
{makeKey("1\xff\xff", 20, 3), makeKey("2")},
{makeKey("\xff"), makeKey("\xff")},
{makeKey("\xff\xff"), makeKey("\xff\xff")},
{makeKey("\xff\xff\xff"), makeKey("\xff\xff\xff")},
{makeKey("\xfe\xff\xff"), makeKey("\xff")},
{makeKey("\xff\xff", 20, 3), makeKey("\xff\xff", 20, 3)},
{makeKey("black"), makeKey("c")},
{makeKey("green"), makeKey("h")},
{makeKey(""), makeKey("")},
{makeKey("13"), makeKey("2")},
{makeKey("135"), makeKey("2")},
{makeKey("13\xff"), makeKey("2")},
{makeKey("1\xff\xff", 20, 3), makeKey("2")},
{makeKey("\xff"), makeKey("\xff")},
{makeKey("\xff\xff"), makeKey("\xff\xff")},
{makeKey("\xff\xff\xff"), makeKey("\xff\xff\xff")},
{makeKey("\xfe\xff\xff"), makeKey("\xff")},
{makeKey("\xff\xff", 20, 3), makeKey("\xff\xff", 20, 3)},
};

for (const auto& c: succCases) {
for (const auto& c : succCases) {
auto a_str = EncodeKey(c.a);
std::printf("a_str: %s\n", a_str.c_str());
comp.FindShortSuccessor(&a_str);
Expand Down
10 changes: 6 additions & 4 deletions c-deps/libroach/db.cc
Original file line number Diff line number Diff line change
Expand Up @@ -940,7 +940,7 @@ DBSstFileWriter* DBSstFileWriterNew() {
// This makes the sstables produced by Pebble and RocksDB byte-by-byte identical, which is
// useful for testing.
table_options.index_shortening =
rocksdb::BlockBasedTableOptions::IndexShorteningMode::kShortenSeparatorsAndSuccessor;
rocksdb::BlockBasedTableOptions::IndexShorteningMode::kShortenSeparatorsAndSuccessor;

rocksdb::Options* options = new rocksdb::Options();
options->comparator = &kComparator;
Expand Down Expand Up @@ -1101,7 +1101,7 @@ DBStatus DBExportToSst(DBKey start, DBKey end, bool export_all_revisions, uint64
std::string resume_key;
// Seek to the MVCC metadata key for the provided start key and let the
// incremental iterator find the appropriate version.
const DBKey seek_key = { .key = start.key };
const DBKey seek_key = {.key = start.key};
for (state = iter.seek(seek_key);; state = iter.next(skip_current_key_versions)) {
if (state.status.data != NULL) {
DBSstFileWriterClose(writer);
Expand All @@ -1128,7 +1128,8 @@ DBStatus DBExportToSst(DBKey start, DBKey end, bool export_all_revisions, uint64

// Skip tombstone (len=0) records when start time is zero (non-incremental)
// and we are not exporting all versions.
const bool is_skipping_deletes = start.wall_time == 0 && start.logical == 0 && !export_all_revisions;
const bool is_skipping_deletes =
start.wall_time == 0 && start.logical == 0 && !export_all_revisions;
if (is_skipping_deletes && iter.value().size() == 0) {
continue;
}
Expand Down Expand Up @@ -1178,7 +1179,8 @@ DBStatus DBEnvOpenReadableFile(DBEngine* db, DBSlice path, DBReadableFile* file)
return db->EnvOpenReadableFile(path, (rocksdb::RandomAccessFile**)file);
}

DBStatus DBEnvReadAtFile(DBEngine* db, DBReadableFile file, DBSlice buffer, int64_t offset, int* n) {
DBStatus DBEnvReadAtFile(DBEngine* db, DBReadableFile file, DBSlice buffer, int64_t offset,
int* n) {
return db->EnvReadAtFile((rocksdb::RandomAccessFile*)file, buffer, offset, n);
}

Expand Down
7 changes: 3 additions & 4 deletions c-deps/libroach/engine.cc
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,8 @@ DBStatus DBImpl::EnvCloseReadableFile(rocksdb::RandomAccessFile* file) {
return kSuccess;
}

DBStatus DBImpl::EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n) {
DBStatus DBImpl::EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset,
int* n) {
size_t max_bytes_to_read = buffer.len;
char* scratch = buffer.data;
rocksdb::Slice result;
Expand All @@ -488,9 +489,7 @@ DBStatus DBImpl::EnvOpenDirectory(DBSlice path, rocksdb::Directory** file) {
return kSuccess;
}

DBStatus DBImpl::EnvSyncDirectory(rocksdb::Directory* file) {
return ToDBStatus(file->Fsync());
}
DBStatus DBImpl::EnvSyncDirectory(rocksdb::Directory* file) { return ToDBStatus(file->Fsync()); }

DBStatus DBImpl::EnvCloseDirectory(rocksdb::Directory* file) {
delete file;
Expand Down
8 changes: 5 additions & 3 deletions c-deps/libroach/engine.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,14 @@ struct DBEngine {
virtual DBStatus EnvDeleteDirAndFiles(DBSlice dir) = 0;
virtual DBStatus EnvLinkFile(DBSlice oldname, DBSlice newname) = 0;
virtual DBStatus EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file) = 0;
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n) = 0;
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset,
int* n) = 0;
virtual DBStatus EnvCloseReadableFile(rocksdb::RandomAccessFile* file) = 0;
virtual DBStatus EnvOpenDirectory(DBSlice path, rocksdb::Directory** file) = 0;
virtual DBStatus EnvSyncDirectory(rocksdb::Directory* file) = 0;
virtual DBStatus EnvCloseDirectory(rocksdb::Directory* file) = 0;
virtual DBStatus EnvRenameFile(DBSlice oldname, DBSlice newname) = 0;

DBSSTable* GetSSTables(int* n);
DBStatus GetSortedWALFiles(DBWALFile** out_files, int* n);
DBString GetUserProperties();
Expand Down Expand Up @@ -107,7 +108,8 @@ struct DBImpl : public DBEngine {
virtual DBStatus EnvDeleteDirAndFiles(DBSlice dir);
virtual DBStatus EnvLinkFile(DBSlice oldname, DBSlice newname);
virtual DBStatus EnvOpenReadableFile(DBSlice path, rocksdb::RandomAccessFile** file);
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset, int* n);
virtual DBStatus EnvReadAtFile(rocksdb::RandomAccessFile* file, DBSlice buffer, int64_t offset,
int* n);
virtual DBStatus EnvCloseReadableFile(rocksdb::RandomAccessFile* file);
virtual DBStatus EnvOpenDirectory(DBSlice path, rocksdb::Directory** file);
virtual DBStatus EnvSyncDirectory(rocksdb::Directory* file);
Expand Down
Loading