From d2d33007cbb9b40ebd1436278ab8b205e3094f76 Mon Sep 17 00:00:00 2001 From: weizuo Date: Thu, 8 Apr 2021 13:18:34 +0800 Subject: [PATCH 1/2] submit base and cumulative compation to different thread pool --- be/src/common/config.h | 9 +- be/src/olap/olap_server.cpp | 151 ++++++++++++++++++++++----------- be/src/olap/storage_engine.cpp | 8 +- be/src/olap/storage_engine.h | 15 ++-- be/src/olap/tablet_manager.cpp | 14 +-- be/src/olap/tablet_manager.h | 3 +- 6 files changed, 135 insertions(+), 65 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index f1547f1180d7b0..e998ec1902f5d6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -283,8 +283,10 @@ CONF_mInt32(cumulative_compaction_skip_window_seconds, "30"); CONF_mInt64(min_compaction_failure_interval_sec, "600"); // 10 min // This config can be set to limit thread number in compaction thread pool. -CONF_mInt32(min_compaction_threads, "10"); -CONF_mInt32(max_compaction_threads, "10"); +CONF_mInt32(min_base_compaction_threads, "10"); +CONF_mInt32(max_base_compaction_threads, "10"); +CONF_mInt32(min_cumulative_compaction_threads, "10"); +CONF_mInt32(max_cumulative_compaction_threads, "10"); // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. CONF_mInt64(total_permits_for_compaction_score, "10000"); @@ -293,7 +295,8 @@ CONF_mInt64(total_permits_for_compaction_score, "10000"); CONF_mInt32(generate_compaction_tasks_min_interval_ms, "10") // Compaction task number per disk. -CONF_mInt32(compaction_task_num_per_disk, "2"); +CONF_mInt32(max_base_compaction_task_num_per_disk, "2"); +CONF_mInt32(max_cumulative_compaction_task_num_per_disk, "2"); // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. CONF_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9"); diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 19d28650758338..777cbc420c4889 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -67,12 +67,19 @@ Status StorageEngine::start_bg_threads() { // check cumulative compaction config _check_cumulative_compaction_config(); - int32_t max_thread_num = config::max_compaction_threads; - int32_t min_thread_num = config::min_compaction_threads; - ThreadPoolBuilder("CompactionTaskThreadPool") - .set_min_threads(min_thread_num) - .set_max_threads(max_thread_num) - .build(&_compaction_thread_pool); + int32_t max_base_thread_num = config::max_base_compaction_threads; + int32_t min_base_thread_num = config::min_base_compaction_threads; + ThreadPoolBuilder("BaseCompactionTaskThreadPool") + .set_min_threads(min_base_thread_num) + .set_max_threads(max_base_thread_num) + .build(&_base_compaction_thread_pool); + + int32_t max_cumulative_thread_num = config::max_cumulative_compaction_threads; + int32_t min_cumulative_thread_num = config::min_cumulative_compaction_threads; + ThreadPoolBuilder("CumulativeCompactionTaskThreadPool") + .set_min_threads(min_cumulative_thread_num) + .set_max_threads(max_cumulative_thread_num) + .build(&_cumulative_compaction_thread_pool); // compaction tasks producer thread RETURN_IF_ERROR(Thread::create( @@ -315,11 +322,18 @@ void StorageEngine::_compaction_tasks_producer_callback() { #endif LOG(INFO) << "try to start compaction producer process!"; - std::vector tablet_submitted; + std::set tablet_submitted; std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); - _tablet_submitted_compaction[tmp_store.second] = tablet_submitted; + { + std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + _tablet_submitted_base_compaction[tmp_store.second] = tablet_submitted; + } + { + std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + _tablet_submitted_cumulative_compaction[tmp_store.second] = tablet_submitted; + } } int round = 0; @@ -355,21 +369,40 @@ void StorageEngine::_compaction_tasks_producer_callback() { for (const auto& tablet : tablets_compaction) { int64_t permits = tablet->prepare_compaction_and_calculate_permits(compaction_type, tablet); if (permits > 0 && _permit_limiter.request(permits)) { - // Push to _tablet_submitted_compaction before submitting task - _push_tablet_into_submitted_compaction(tablet); - auto st =_compaction_thread_pool->submit_func([=]() { - CgroupsMgr::apply_system_cgroup(); - tablet->execute_compaction(compaction_type); - _permit_limiter.release(permits); - _pop_tablet_from_submitted_compaction(tablet); - // reset compaction - tablet->reset_compaction(compaction_type); - }); - if (!st.ok()) { - _permit_limiter.release(permits); - _pop_tablet_from_submitted_compaction(tablet); - // reset compaction - tablet->reset_compaction(compaction_type); + if (compaction_type == CompactionType::BASE_COMPACTION) { + // Push to _tablet_submitted_base_compaction before submitting task + _push_tablet_into_submitted_base_compaction(tablet); + auto st =_base_compaction_thread_pool->submit_func([=]() { + CgroupsMgr::apply_system_cgroup(); + tablet->execute_compaction(compaction_type); + _permit_limiter.release(permits); + _pop_tablet_from_submitted_base_compaction(tablet); + // reset compaction + tablet->reset_compaction(compaction_type); + }); + if (!st.ok()) { + _permit_limiter.release(permits); + _pop_tablet_from_submitted_base_compaction(tablet); + // reset compaction + tablet->reset_compaction(compaction_type); + } + } else { + // Push to _tablet_submitted_cumulative_compaction before submitting task + _push_tablet_into_submitted_cumulative_compaction(tablet); + auto st =_cumulative_compaction_thread_pool->submit_func([=]() { + CgroupsMgr::apply_system_cgroup(); + tablet->execute_compaction(compaction_type); + _permit_limiter.release(permits); + _pop_tablet_from_submitted_cumulative_compaction(tablet); + // reset compaction + tablet->reset_compaction(compaction_type); + }); + if (!st.ok()) { + _permit_limiter.release(permits); + _pop_tablet_from_submitted_cumulative_compaction(tablet); + // reset compaction + tablet->reset_compaction(compaction_type); + } } } else { // reset compaction @@ -387,41 +420,61 @@ std::vector StorageEngine::_compaction_tasks_generator( CompactionType compaction_type, std::vector data_dirs) { std::vector tablets_compaction; std::random_shuffle(data_dirs.begin(), data_dirs.end()); - for (auto data_dir : data_dirs) { - std::unique_lock lock(_tablet_submitted_compaction_mutex); - if (_tablet_submitted_compaction[data_dir].size() >= config::compaction_task_num_per_disk) { - continue; + if (compaction_type == CompactionType::BASE_COMPACTION) { + for (auto data_dir : data_dirs) { + std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + if (_tablet_submitted_base_compaction[data_dir].size() >= config::max_base_compaction_task_num_per_disk) { + continue; + } + if (!data_dir->reach_capacity_limit(0)) { + std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction( + compaction_type, data_dir, _tablet_submitted_base_compaction[data_dir], + _tablet_submitted_cumulative_compaction[data_dir]); + if (tablet != nullptr) { + tablets_compaction.emplace_back(tablet); + } + } } - if (!data_dir->reach_capacity_limit(0)) { - TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction( - compaction_type, data_dir, _tablet_submitted_compaction[data_dir]); - if (tablet != nullptr) { - tablets_compaction.emplace_back(tablet); + } else { + for (auto data_dir : data_dirs) { + std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + if (_tablet_submitted_cumulative_compaction[data_dir].size() >= config::max_cumulative_compaction_task_num_per_disk) { + continue; + } + if (!data_dir->reach_capacity_limit(0)) { + std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction( + compaction_type, data_dir, _tablet_submitted_base_compaction[data_dir], + _tablet_submitted_cumulative_compaction[data_dir]); + if (tablet != nullptr) { + tablets_compaction.emplace_back(tablet); + } } } } + return tablets_compaction; } -void StorageEngine::_push_tablet_into_submitted_compaction(TabletSharedPtr tablet) { - std::unique_lock lock(_tablet_submitted_compaction_mutex); - _tablet_submitted_compaction[tablet->data_dir()].emplace_back( - tablet->tablet_id()); +void StorageEngine::_push_tablet_into_submitted_base_compaction(TabletSharedPtr tablet) { + std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + _tablet_submitted_base_compaction[tablet->data_dir()].insert(tablet->tablet_id()); } -void StorageEngine::_pop_tablet_from_submitted_compaction(TabletSharedPtr tablet) { - std::unique_lock lock(_tablet_submitted_compaction_mutex); - std::vector::iterator it_tablet = - find(_tablet_submitted_compaction[tablet->data_dir()].begin(), - _tablet_submitted_compaction[tablet->data_dir()].end(), - tablet->tablet_id()); - if (it_tablet != - _tablet_submitted_compaction[tablet->data_dir()].end()) { - _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); - std::unique_lock lock(_compaction_producer_sleep_mutex); - _wakeup_producer_flag = 1; - _compaction_producer_sleep_cv.notify_one(); - } +void StorageEngine::_pop_tablet_from_submitted_base_compaction(TabletSharedPtr tablet) { + std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + _tablet_submitted_base_compaction[tablet->data_dir()].erase(tablet->tablet_id()); +} + +void StorageEngine::_push_tablet_into_submitted_cumulative_compaction(TabletSharedPtr tablet) { + std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + _tablet_submitted_cumulative_compaction[tablet->data_dir()].insert(tablet->tablet_id()); +} + +void StorageEngine::_pop_tablet_from_submitted_cumulative_compaction(TabletSharedPtr tablet) { + std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + _tablet_submitted_cumulative_compaction[tablet->data_dir()].erase(tablet->tablet_id()); } } // namespace doris diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 670cd2103243bd..9a6e112a5918d5 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -140,8 +140,12 @@ StorageEngine::~StorageEngine() { DEREGISTER_HOOK_METRIC(compaction_mem_current_consumption); _clear(); - if(_compaction_thread_pool){ - _compaction_thread_pool->shutdown(); + if(_base_compaction_thread_pool){ + _base_compaction_thread_pool->shutdown(); + } + + if(_cumulative_compaction_thread_pool){ + _cumulative_compaction_thread_pool->shutdown(); } } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 04964c96023ee4..7377a15d524dfd 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -239,8 +239,10 @@ class StorageEngine { void _compaction_tasks_producer_callback(); vector _compaction_tasks_generator(CompactionType compaction_type, std::vector data_dirs); - void _push_tablet_into_submitted_compaction(TabletSharedPtr tablet); - void _pop_tablet_from_submitted_compaction(TabletSharedPtr tablet); + void _push_tablet_into_submitted_base_compaction(TabletSharedPtr tablet); + void _pop_tablet_from_submitted_base_compaction(TabletSharedPtr tablet); + void _push_tablet_into_submitted_cumulative_compaction(TabletSharedPtr tablet); + void _pop_tablet_from_submitted_cumulative_compaction(TabletSharedPtr tablet); private: struct CompactionCandidate { @@ -334,12 +336,15 @@ class StorageEngine { HeartbeatFlags* _heartbeat_flags; - std::unique_ptr _compaction_thread_pool; + std::unique_ptr _base_compaction_thread_pool; + std::unique_ptr _cumulative_compaction_thread_pool; CompactionPermitLimiter _permit_limiter; - std::mutex _tablet_submitted_compaction_mutex; - std::map> _tablet_submitted_compaction; + std::mutex _tablet_submitted_base_compaction_mutex; + std::map> _tablet_submitted_base_compaction; + std::mutex _tablet_submitted_cumulative_compaction_mutex; + std::map> _tablet_submitted_cumulative_compaction; AtomicInt32 _wakeup_producer_flag; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index df81c8bcc1b8a5..b9235154d903da 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -681,7 +681,8 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) { TabletSharedPtr TabletManager::find_best_tablet_to_compaction( CompactionType compaction_type, DataDir* data_dir, - std::vector& tablet_submitted_compaction) { + std::set& tablet_submitted_base_compaction, + std::set& tablet_submitted_cumulative_compaction) { int64_t now_ms = UnixMillis(); const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative"; @@ -693,10 +694,13 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( ReadLock rlock(tablets_shard.lock.get()); for (const auto& tablet_map : tablets_shard.tablet_map) { for (const TabletSharedPtr& tablet_ptr : tablet_map.second.table_arr) { - std::vector::iterator it_tablet = - find(tablet_submitted_compaction.begin(), tablet_submitted_compaction.end(), - tablet_ptr->tablet_id()); - if (it_tablet != tablet_submitted_compaction.end()) { + std::set::iterator it_tablet = + tablet_submitted_base_compaction.find(tablet_ptr->tablet_id()); + if (it_tablet != tablet_submitted_base_compaction.end()) { + continue; + } + it_tablet = tablet_submitted_cumulative_compaction.find(tablet_ptr->tablet_id()); + if (it_tablet != tablet_submitted_cumulative_compaction.end()) { continue; } AlterTabletTaskSharedPtr cur_alter_task = tablet_ptr->alter_task(); diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index bbb417d1a28a93..7be193b800fe1d 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -71,7 +71,8 @@ class TabletManager { TabletSharedPtr find_best_tablet_to_compaction(CompactionType compaction_type, DataDir* data_dir, - vector& tablet_submitted_compaction); + std::set& tablet_submitted_base_compaction, + std::set& tablet_submitted_cumulative_compaction); TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash, bool include_deleted = false, std::string* err = nullptr); From 3717b008b0ca310cc9e4887fdd94b3aef41f163b Mon Sep 17 00:00:00 2001 From: weizuo Date: Thu, 8 Apr 2021 15:33:37 +0800 Subject: [PATCH 2/2] submit base and cumulative compation to different thread pool --- be/src/olap/olap_server.cpp | 30 +++++++++++++++--------------- be/src/olap/storage_engine.h | 4 ++-- be/src/olap/tablet_manager.cpp | 13 +++++++------ be/src/olap/tablet_manager.h | 4 ++-- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 777cbc420c4889..a6d305ac92230d 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -327,11 +327,11 @@ void StorageEngine::_compaction_tasks_producer_callback() { for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); { - std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + WriteLock wr_lock(&_tablet_submitted_base_compaction_mutex); _tablet_submitted_base_compaction[tmp_store.second] = tablet_submitted; } { - std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + WriteLock wr_lock(&_tablet_submitted_cumulative_compaction_mutex); _tablet_submitted_cumulative_compaction[tmp_store.second] = tablet_submitted; } } @@ -349,6 +349,7 @@ void StorageEngine::_compaction_tasks_producer_callback() { compaction_type = CompactionType::BASE_COMPACTION; round = 0; } + std::vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); if (tablets_compaction.size() == 0) { @@ -422,15 +423,15 @@ std::vector StorageEngine::_compaction_tasks_generator( std::random_shuffle(data_dirs.begin(), data_dirs.end()); if (compaction_type == CompactionType::BASE_COMPACTION) { for (auto data_dir : data_dirs) { - std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + ReadLock rd_lock(&_tablet_submitted_base_compaction_mutex); if (_tablet_submitted_base_compaction[data_dir].size() >= config::max_base_compaction_task_num_per_disk) { continue; } if (!data_dir->reach_capacity_limit(0)) { - std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + ReadLock rd_lock(&_tablet_submitted_cumulative_compaction_mutex); TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction( - compaction_type, data_dir, _tablet_submitted_base_compaction[data_dir], - _tablet_submitted_cumulative_compaction[data_dir]); + compaction_type, data_dir, &_tablet_submitted_base_compaction[data_dir], + &_tablet_submitted_cumulative_compaction[data_dir]); if (tablet != nullptr) { tablets_compaction.emplace_back(tablet); } @@ -438,42 +439,41 @@ std::vector StorageEngine::_compaction_tasks_generator( } } else { for (auto data_dir : data_dirs) { - std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + ReadLock rd_lock(&_tablet_submitted_cumulative_compaction_mutex); if (_tablet_submitted_cumulative_compaction[data_dir].size() >= config::max_cumulative_compaction_task_num_per_disk) { continue; } if (!data_dir->reach_capacity_limit(0)) { - std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + ReadLock rd_lock(&_tablet_submitted_base_compaction_mutex); TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction( - compaction_type, data_dir, _tablet_submitted_base_compaction[data_dir], - _tablet_submitted_cumulative_compaction[data_dir]); + compaction_type, data_dir, &_tablet_submitted_base_compaction[data_dir], + &_tablet_submitted_cumulative_compaction[data_dir]); if (tablet != nullptr) { tablets_compaction.emplace_back(tablet); } } } } - return tablets_compaction; } void StorageEngine::_push_tablet_into_submitted_base_compaction(TabletSharedPtr tablet) { - std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + WriteLock wr_lock(&_tablet_submitted_base_compaction_mutex); _tablet_submitted_base_compaction[tablet->data_dir()].insert(tablet->tablet_id()); } void StorageEngine::_pop_tablet_from_submitted_base_compaction(TabletSharedPtr tablet) { - std::unique_lock lock(_tablet_submitted_base_compaction_mutex); + WriteLock wr_lock(&_tablet_submitted_base_compaction_mutex); _tablet_submitted_base_compaction[tablet->data_dir()].erase(tablet->tablet_id()); } void StorageEngine::_push_tablet_into_submitted_cumulative_compaction(TabletSharedPtr tablet) { - std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + WriteLock wr_lock(&_tablet_submitted_cumulative_compaction_mutex); _tablet_submitted_cumulative_compaction[tablet->data_dir()].insert(tablet->tablet_id()); } void StorageEngine::_pop_tablet_from_submitted_cumulative_compaction(TabletSharedPtr tablet) { - std::unique_lock lock(_tablet_submitted_cumulative_compaction_mutex); + WriteLock wr_lock(&_tablet_submitted_cumulative_compaction_mutex); _tablet_submitted_cumulative_compaction[tablet->data_dir()].erase(tablet->tablet_id()); } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 7377a15d524dfd..3d2e94a365fe20 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -341,9 +341,9 @@ class StorageEngine { CompactionPermitLimiter _permit_limiter; - std::mutex _tablet_submitted_base_compaction_mutex; + RWMutex _tablet_submitted_base_compaction_mutex; std::map> _tablet_submitted_base_compaction; - std::mutex _tablet_submitted_cumulative_compaction_mutex; + RWMutex _tablet_submitted_cumulative_compaction_mutex; std::map> _tablet_submitted_cumulative_compaction; AtomicInt32 _wakeup_producer_flag; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index b9235154d903da..dfa7fb8cfb1e7d 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -681,8 +681,8 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) { TabletSharedPtr TabletManager::find_best_tablet_to_compaction( CompactionType compaction_type, DataDir* data_dir, - std::set& tablet_submitted_base_compaction, - std::set& tablet_submitted_cumulative_compaction) { + std::set* tablet_submitted_base_compaction, + std::set* tablet_submitted_cumulative_compaction) { int64_t now_ms = UnixMillis(); const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative"; @@ -695,14 +695,15 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( for (const auto& tablet_map : tablets_shard.tablet_map) { for (const TabletSharedPtr& tablet_ptr : tablet_map.second.table_arr) { std::set::iterator it_tablet = - tablet_submitted_base_compaction.find(tablet_ptr->tablet_id()); - if (it_tablet != tablet_submitted_base_compaction.end()) { + (*tablet_submitted_base_compaction).find(tablet_ptr->tablet_id()); + if (it_tablet != (*tablet_submitted_base_compaction).end()) { continue; } - it_tablet = tablet_submitted_cumulative_compaction.find(tablet_ptr->tablet_id()); - if (it_tablet != tablet_submitted_cumulative_compaction.end()) { + it_tablet = (*tablet_submitted_cumulative_compaction).find(tablet_ptr->tablet_id()); + if (it_tablet != (*tablet_submitted_cumulative_compaction).end()) { continue; } + AlterTabletTaskSharedPtr cur_alter_task = tablet_ptr->alter_task(); if (cur_alter_task != nullptr && cur_alter_task->alter_state() != ALTER_FINISHED && cur_alter_task->alter_state() != ALTER_FAILED) { diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 7be193b800fe1d..8fca44bed26595 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -71,8 +71,8 @@ class TabletManager { TabletSharedPtr find_best_tablet_to_compaction(CompactionType compaction_type, DataDir* data_dir, - std::set& tablet_submitted_base_compaction, - std::set& tablet_submitted_cumulative_compaction); + std::set* tablet_submitted_base_compaction, + std::set* tablet_submitted_cumulative_compaction); TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash, bool include_deleted = false, std::string* err = nullptr);