Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Group SST write in flush, compaction and db open with new stats #11910

Closed
wants to merge 1 commit into from
+1,822 −1,048
Closed

Group SST write in flush, compaction and db open with new stats #11910

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions db/blob/blob_file_builder.cc
View file
Original file line number Diff line number Diff line change
@@ -34,28 +34,28 @@ BlobFileBuilder::BlobFileBuilder(
VersionSet* versions, FileSystem* fs,
const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
std::string db_id, std::string db_session_id, int job_id,
uint32_t column_family_id, const std::string& column_family_name,
Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
BlobFileCompletionCallback* blob_callback,
BlobFileCreationReason creation_reason,
std::vector<std::string>* blob_file_paths,
std::vector<BlobFileAddition>* blob_file_additions)
: BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
immutable_options, mutable_cf_options, file_options,
db_id, db_session_id, job_id, column_family_id,
column_family_name, io_priority, write_hint, io_tracer,
blob_callback, creation_reason, blob_file_paths,
blob_file_additions) {}
write_options, db_id, db_session_id, job_id,
column_family_id, column_family_name, write_hint,
io_tracer, blob_callback, creation_reason,
blob_file_paths, blob_file_additions) {}

BlobFileBuilder::BlobFileBuilder(
std::function<uint64_t()> file_number_generator, FileSystem* fs,
const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
std::string db_id, std::string db_session_id, int job_id,
uint32_t column_family_id, const std::string& column_family_name,
Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
BlobFileCompletionCallback* blob_callback,
BlobFileCreationReason creation_reason,
@@ -69,12 +69,12 @@ BlobFileBuilder::BlobFileBuilder(
blob_compression_type_(mutable_cf_options->blob_compression_type),
prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache),
file_options_(file_options),
write_options_(write_options),
db_id_(std::move(db_id)),
db_session_id_(std::move(db_session_id)),
job_id_(job_id),
column_family_id_(column_family_id),
column_family_name_(column_family_name),
io_priority_(io_priority),
write_hint_(write_hint),
io_tracer_(io_tracer),
blob_callback_(blob_callback),
@@ -87,6 +87,7 @@ BlobFileBuilder::BlobFileBuilder(
assert(fs_);
assert(immutable_options_);
assert(file_options_);
assert(write_options_);
assert(blob_file_paths_);
assert(blob_file_paths_->empty());
assert(blob_file_additions_);
@@ -207,14 +208,14 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
blob_file_paths_->emplace_back(std::move(blob_file_path));

assert(file);
file->SetIOPriority(io_priority_);
file->SetIOPriority(write_options_->rate_limiter_priority);
file->SetWriteLifeTimeHint(write_hint_);
FileTypeSet tmp_set = immutable_options_->checksum_handoff_file_types;
Statistics* const statistics = immutable_options_->stats;
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(file), blob_file_paths_->back(), *file_options_,
immutable_options_->clock, io_tracer_, statistics,
immutable_options_->listeners,
Histograms::BLOB_DB_BLOB_FILE_WRITE_MICROS, immutable_options_->listeners,
immutable_options_->file_checksum_gen_factory.get(),
tmp_set.Contains(FileType::kBlobFile), false));

@@ -231,7 +232,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
expiration_range);

{
Status s = blob_log_writer->WriteHeader(header);
Status s = blob_log_writer->WriteHeader(*write_options_, header);

TEST_SYNC_POINT_CALLBACK(
"BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
@@ -296,7 +297,8 @@ Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob,

uint64_t key_offset = 0;

Status s = writer_->AddRecord(key, blob, &key_offset, blob_offset);
Status s =
writer_->AddRecord(*write_options_, key, blob, &key_offset, blob_offset);

TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s);

@@ -321,7 +323,8 @@ Status BlobFileBuilder::CloseBlobFile() {
std::string checksum_method;
std::string checksum_value;

Status s = writer_->AppendFooter(footer, &checksum_method, &checksum_value);
Status s = writer_->AppendFooter(*write_options_, footer, &checksum_method,
&checksum_value);

TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s);

11 changes: 6 additions & 5 deletions db/blob/blob_file_builder.h
View file
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@
#include "rocksdb/advanced_options.h"
#include "rocksdb/compression_type.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/rocksdb_namespace.h"
#include "rocksdb/types.h"

@@ -36,11 +37,11 @@ class BlobFileBuilder {
BlobFileBuilder(VersionSet* versions, FileSystem* fs,
const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options,
const FileOptions* file_options, std::string db_id,
const FileOptions* file_options,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id,
uint32_t column_family_id,
const std::string& column_family_name,
Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
BlobFileCompletionCallback* blob_callback,
@@ -51,11 +52,11 @@ class BlobFileBuilder {
BlobFileBuilder(std::function<uint64_t()> file_number_generator,
FileSystem* fs, const ImmutableOptions* immutable_options,
const MutableCFOptions* mutable_cf_options,
const FileOptions* file_options, std::string db_id,
const FileOptions* file_options,
const WriteOptions* write_options, std::string db_id,
std::string db_session_id, int job_id,
uint32_t column_family_id,
const std::string& column_family_name,
Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint,
const std::shared_ptr<IOTracer>& io_tracer,
BlobFileCompletionCallback* blob_callback,
@@ -92,12 +93,12 @@ class BlobFileBuilder {
CompressionType blob_compression_type_;
PrepopulateBlobCache prepopulate_blob_cache_;
const FileOptions* file_options_;
const WriteOptions* write_options_;
const std::string db_id_;
const std::string db_session_id_;
int job_id_;
uint32_t column_family_id_;
std::string column_family_name_;
Env::IOPriority io_priority_;
Env::WriteLifeTimeHint write_hint_;
std::shared_ptr<IOTracer> io_tracer_;
BlobFileCompletionCallback* blob_callback_;
39 changes: 18 additions & 21 deletions db/blob/blob_file_builder_test.cc
View file
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@ class BlobFileBuilderTest : public testing::Test {
mock_env_.reset(MockEnv::Create(Env::Default()));
fs_ = mock_env_->GetFileSystem().get();
clock_ = mock_env_->GetSystemClock().get();
write_options_.rate_limiter_priority = Env::IO_HIGH;
}

void VerifyBlobFile(uint64_t blob_file_number,
@@ -113,6 +114,7 @@ class BlobFileBuilderTest : public testing::Test {
FileSystem* fs_;
SystemClock* clock_;
FileOptions file_options_;
WriteOptions write_options_;
};

TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
@@ -136,16 +138,15 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

@@ -221,16 +222,15 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

@@ -309,16 +309,15 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

@@ -364,16 +363,15 @@ TEST_F(BlobFileBuilderTest, Compression) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

@@ -448,16 +446,15 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

@@ -528,16 +525,15 @@ TEST_F(BlobFileBuilderTest, Checksum) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

@@ -589,11 +585,13 @@ class BlobFileBuilderIOErrorTest
BlobFileBuilderIOErrorTest() : sync_point_(GetParam()) {
mock_env_.reset(MockEnv::Create(Env::Default()));
fs_ = mock_env_->GetFileSystem().get();
write_options_.rate_limiter_priority = Env::IO_HIGH;
}

std::unique_ptr<Env> mock_env_;
FileSystem* fs_;
FileOptions file_options_;
WriteOptions write_options_;
std::string sync_point_;
};

@@ -626,16 +624,15 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;

std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions;

BlobFileBuilder builder(
TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
&file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
column_family_id, column_family_name, io_priority, write_hint,
&file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/,
job_id, column_family_id, column_family_name, write_hint,
nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);

9 changes: 5 additions & 4 deletions db/blob/blob_file_cache_test.cc
View file
Original file line number Diff line number Diff line change
@@ -57,7 +57,7 @@ void WriteBlobFile(uint32_t column_family_id,
BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
expiration_range);

ASSERT_OK(blob_log_writer.WriteHeader(header));
ASSERT_OK(blob_log_writer.WriteHeader(WriteOptions(), header));

constexpr char key[] = "key";
constexpr char blob[] = "blob";
@@ -67,7 +67,8 @@ void WriteBlobFile(uint32_t column_family_id,
uint64_t key_offset = 0;
uint64_t blob_offset = 0;

ASSERT_OK(blob_log_writer.AddRecord(key, blob, &key_offset, &blob_offset));
ASSERT_OK(blob_log_writer.AddRecord(WriteOptions(), key, blob, &key_offset,
&blob_offset));

BlobLogFooter footer;
footer.blob_count = 1;
@@ -76,8 +77,8 @@ void WriteBlobFile(uint32_t column_family_id,
std::string checksum_method;
std::string checksum_value;

ASSERT_OK(
blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
&checksum_method, &checksum_value));
}

} // anonymous namespace
11 changes: 6 additions & 5 deletions db/blob/blob_file_reader_test.cc
View file
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,
BlobLogHeader header(column_family_id, compression, has_ttl,
expiration_range_header);

ASSERT_OK(blob_log_writer.WriteHeader(header));
ASSERT_OK(blob_log_writer.WriteHeader(WriteOptions(), header));

std::vector<std::string> compressed_blobs(num);
std::vector<Slice> blobs_to_write(num);
@@ -91,7 +91,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,

for (size_t i = 0; i < num; ++i) {
uint64_t key_offset = 0;
ASSERT_OK(blob_log_writer.AddRecord(keys[i], blobs_to_write[i], &key_offset,
ASSERT_OK(blob_log_writer.AddRecord(WriteOptions(), keys[i],
blobs_to_write[i], &key_offset,
&blob_offsets[i]));
}

@@ -101,8 +102,8 @@ void WriteBlobFile(const ImmutableOptions& immutable_options,

std::string checksum_method;
std::string checksum_value;
ASSERT_OK(
blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
ASSERT_OK(blob_log_writer.AppendFooter(WriteOptions(), footer,
&checksum_method, &checksum_value));
}

// Creates a test blob file with a single blob in it. Note: this method
@@ -473,7 +474,7 @@ TEST_F(BlobFileReaderTest, Malformed) {
BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
expiration_range);

ASSERT_OK(blob_log_writer.WriteHeader(header));
ASSERT_OK(blob_log_writer.WriteHeader(WriteOptions(), header));
}

constexpr HistogramImpl* blob_file_read_hist = nullptr;
Loading