Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Group rocksdb.sst.read.micros stat by IOActivity flush and compaction #11288

Closed
wants to merge 17 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix some read options
hx235 committed Apr 20, 2023
commit 78854356fe05f1b6b6ed13cd0dffed56db3790ba
3 changes: 2 additions & 1 deletion db/compaction/compaction_job.cc
Original file line number Diff line number Diff line change
@@ -618,6 +618,8 @@ Status CompactionJob::Run() {
AutoThreadOperationStageUpdater stage_updater(
ThreadStatus::STAGE_COMPACTION_RUN);
ThreadIOActivityGuard thread_io_activity_guard(Env::IOActivity::kCompaction);
const ReadOptions read_options(Env::IOActivity::kCompaction);

TEST_SYNC_POINT("CompactionJob::Run():Start");
log_buffer_->FlushBufferToLog();
LogCompaction();
@@ -726,7 +728,6 @@ Status CompactionJob::Run() {
// use_direct_io_for_flush_and_compaction is true, we will regard this
// verification as user reads since the goal is to cache it here for
// further user reads
const ReadOptions read_options(Env::IOActivity::kCompaction);
InternalIterator* iter = cfd->table_cache()->NewIterator(
read_options, file_options_, cfd->internal_comparator(),
files_output[file_idx]->meta, /*range_del_agg=*/nullptr,
4 changes: 2 additions & 2 deletions db/compaction/compaction_job_test.cc
Original file line number Diff line number Diff line change
@@ -316,7 +316,6 @@ class CompactionJobTestBase : public testing::Test {
void AddMockFile(const mock::KVVector& contents, int level = 0) {
assert(contents.size() > 0);

const ReadOptions read_options;
bool first_key = true;
std::string smallest, largest;
InternalKey smallest_key, largest_key;
@@ -392,7 +391,7 @@ class CompactionJobTestBase : public testing::Test {
mutex_.Lock();
EXPECT_OK(versions_->LogAndApply(
versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_,
read_options, &edit, &mutex_, nullptr));
read_options_, &edit, &mutex_, nullptr));
mutex_.Unlock();
}

@@ -728,6 +727,7 @@ class CompactionJobTestBase : public testing::Test {
ColumnFamilyOptions cf_options_;
MutableCFOptions mutable_cf_options_;
MutableDBOptions mutable_db_options_;
const ReadOptions read_options_;
std::shared_ptr<Cache> table_cache_;
WriteController write_controller_;
WriteBufferManager write_buffer_manager_;
5 changes: 3 additions & 2 deletions db/repair.cc
Original file line number Diff line number Diff line change
@@ -360,6 +360,9 @@ class Repairer {
}
};

// TODO: plumb Env::IOActivity
const ReadOptions read_options;

// Open the log file
std::string logname = LogFileName(wal_dir, log);
const auto& fs = env_->GetFileSystem();
@@ -457,8 +460,6 @@ class Repairer {
0 /*target_file_size*/, meta.fd.GetNumber());

SeqnoToTimeMapping empty_seqno_time_mapping;
// TODO: plumb Env::IOActivity
const ReadOptions read_options;
status = BuildTable(
dbname_, /* versions */ nullptr, immutable_db_options_, tboptions,
file_options_, read_options, table_cache_.get(), iter.get(),
25 changes: 14 additions & 11 deletions db/version_set.cc
Original file line number Diff line number Diff line change
@@ -5695,6 +5695,7 @@ Status VersionSet::Recover(
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
std::string* db_id, bool no_error_if_files_missing) {
ThreadIOActivityGuard thread_io_activity_guard(Env::IOActivity::kRecovery);
const ReadOptions read_options(Env::IOActivity::kRecovery);
// Read "CURRENT" file, which contains a pointer to the current manifest
// file
std::string manifest_path;
@@ -5728,7 +5729,6 @@ Status VersionSet::Recover(
reporter.status = &log_read_status;
log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter,
true /* checksum */, 0 /* log_number */);
const ReadOptions read_options(Env::IOActivity::kRecovery);
VersionEditHandler handler(
read_only, column_families, const_cast<VersionSet*>(this),
/*track_missing_files=*/false, no_error_if_files_missing, io_tracer_,
@@ -5881,6 +5881,7 @@ Status VersionSet::TryRecoverFromOneManifest(
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
std::string* db_id, bool* has_missing_table_file) {
ThreadIOActivityGuard thread_io_activity_guard(Env::IOActivity::kRecovery);
const ReadOptions read_options(Env::IOActivity::kRecovery);
ROCKS_LOG_INFO(db_options_->info_log, "Trying to recover from manifest: %s\n",
manifest_path.c_str());
std::unique_ptr<SequentialFileReader> manifest_file_reader;
@@ -5903,7 +5904,6 @@ Status VersionSet::TryRecoverFromOneManifest(
reporter.status = &s;
log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter,
/*checksum=*/true, /*log_num=*/0);
const ReadOptions read_options(Env::IOActivity::kRecovery);
VersionEditHandlerPointInTime handler_pit(
read_only, column_families, const_cast<VersionSet*>(this), io_tracer_,
EpochNumberRequirement::kMightMissing);
@@ -5949,6 +5949,8 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
Status VersionSet::ListColumnFamiliesFromManifest(
const std::string& manifest_path, FileSystem* fs,
std::vector<std::string>* column_families) {
// TODO: plumb Env::IOActivity
const ReadOptions read_options;
std::unique_ptr<SequentialFileReader> file_reader;
Status s;
{
@@ -5967,8 +5969,6 @@ Status VersionSet::ListColumnFamiliesFromManifest(
reporter.status = &s;
log::Reader reader(nullptr, std::move(file_reader), &reporter,
true /* checksum */, 0 /* log_number */);
// TODO: plumb Env::IOActivity
const ReadOptions read_options;

ListColumnFamiliesHandler handler;
handler.Iterate(read_options, reader, &s);
@@ -5993,6 +5993,9 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
"Number of levels needs to be bigger than 1");
}

// TODO: plumb Env::IOActivity
const ReadOptions read_options;

ImmutableDBOptions db_options(*options);
ColumnFamilyOptions cf_options(*options);
std::shared_ptr<Cache> tc(NewLRUCache(options->max_open_files - 10,
@@ -6076,8 +6079,6 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
vstorage->ResizeCompactCursors(new_levels);

MutableCFOptions mutable_cf_options(*options);
// TODO: plumb Env::IOActivity
const ReadOptions read_options;
VersionEdit ve;
InstrumentedMutex dummy_mutex;
InstrumentedMutexLock l(&dummy_mutex);
@@ -6156,6 +6157,9 @@ Status VersionSet::GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) {
Status VersionSet::DumpManifest(Options& options, std::string& dscname,
bool verbose, bool hex, bool json) {
assert(options.env);
// TODO: plumb Env::IOActivity
const ReadOptions read_options;

std::vector<std::string> column_families;
Status s = ListColumnFamiliesFromManifest(
dscname, options.env->GetFileSystem().get(), &column_families);
@@ -6188,8 +6192,6 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
reporter.status = &s;
log::Reader reader(nullptr, std::move(file_reader), &reporter,
true /* checksum */, 0 /* log_number */);
// TODO: plumb Env::IOActivity
const ReadOptions read_options;
handler.Iterate(read_options, reader, &s);
}

@@ -7018,6 +7020,8 @@ Status ReactiveVersionSet::Recover(
assert(manifest_reader_status != nullptr);

ThreadIOActivityGuard thread_io_activity_guard(Env::IOActivity::kRecovery);
const ReadOptions read_options(Env::IOActivity::kRecovery);

manifest_reader_status->reset(new Status());
manifest_reporter->reset(new LogReporter());
static_cast_with_check<LogReporter>(manifest_reporter->get())->status =
@@ -7028,7 +7032,6 @@ Status ReactiveVersionSet::Recover(
}
log::Reader* reader = manifest_reader->get();
assert(reader);
const ReadOptions read_options(Env::IOActivity::kRecovery);

manifest_tailer_.reset(
new ManifestTailer(column_families, const_cast<ReactiveVersionSet*>(this),
@@ -7052,6 +7055,8 @@ Status ReactiveVersionSet::ReadAndApply(
assert(manifest_reader != nullptr);
assert(cfds_changed != nullptr);
mu->AssertHeld();
// TODO: plumb Env::IOActivity
const ReadOptions read_options;

Status s;
log::Reader* reader = manifest_reader->get();
@@ -7060,8 +7065,6 @@ Status ReactiveVersionSet::ReadAndApply(
if (!s.ok()) {
return s;
}
// TODO: plumb Env::IOActivity
const ReadOptions read_options;
manifest_tailer_->Iterate(read_options, *(manifest_reader->get()),
manifest_read_status);
s = manifest_tailer_->status();
3 changes: 1 addition & 2 deletions db/version_set_test.cc
Original file line number Diff line number Diff line change
@@ -1619,7 +1619,6 @@ TEST_F(VersionSetTest, ObsoleteBlobFile) {
constexpr uint64_t total_blob_bytes = 66666;
constexpr char checksum_method[] = "CRC32";
constexpr char checksum_value[] = "\x3d\x87\xff\x57";
const ReadOptions read_options;

edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value);
@@ -1629,7 +1628,7 @@ TEST_F(VersionSetTest, ObsoleteBlobFile) {
mutex_.Lock();
Status s = versions_->LogAndApply(
versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_,
read_options, &edit, &mutex_, nullptr);
read_options_, &edit, &mutex_, nullptr);
mutex_.Unlock();

ASSERT_OK(s);