diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index c1dbc8a93948df..d2fc09455143d5 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1085,21 +1085,21 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) { return !(a == b); } -DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) { - // The default delete bitmap cache is set to 100MB, - // which can be insufficient and cause performance issues when the amount of user data is large. - // To mitigate the problem of an inadequate cache, - // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. - bool is_percent = false; - int64_t delete_bitmap_agg_cache_cache_limit = - ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit, - MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent); - _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit > - config::delete_bitmap_agg_cache_capacity - ? delete_bitmap_agg_cache_cache_limit - : config::delete_bitmap_agg_cache_capacity)); +DeleteBitmapAggCache::DeleteBitmapAggCache(size_t capacity) + : LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity, + LRUCacheType::SIZE, config::delete_bitmap_agg_cache_stale_sweep_time_sec, + 256) {} + +DeleteBitmapAggCache* DeleteBitmapAggCache::instance() { + return ExecEnv::GetInstance()->delete_bitmap_agg_cache(); +} + +DeleteBitmapAggCache* DeleteBitmapAggCache::create_instance(size_t capacity) { + return new DeleteBitmapAggCache(capacity); } +DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {} + DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) { delete_bitmap = o.delete_bitmap; // just copy data _tablet_id = o._tablet_id; @@ -1438,20 +1438,20 @@ static std::string agg_cache_key(int64_t tablet_id, const DeleteBitmap::BitmapKe std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) const { std::string key_str = agg_cache_key(_tablet_id, bmk); // Cache key container CacheKey key(key_str); - Cache::Handle* handle = _agg_cache->repr()->lookup(key); + Cache::Handle* handle = DeleteBitmapAggCache::instance()->lookup(key); - AggCache::Value* val = - handle == nullptr - ? nullptr - : reinterpret_cast(_agg_cache->repr()->value(handle)); + DeleteBitmapAggCache::Value* val = + handle == nullptr ? nullptr + : reinterpret_cast( + DeleteBitmapAggCache::instance()->value(handle)); // FIXME: do we need a mutex here to get rid of duplicated initializations // of cache entries in some cases? if (val == nullptr) { // Renew if needed, put a new Value to cache - val = new AggCache::Value(); + val = new DeleteBitmapAggCache::Value(); Version start_version = config::enable_mow_get_agg_by_cache ? _get_rowset_cache_version(bmk) : 0; if (start_version > 0) { - Cache::Handle* handle2 = _agg_cache->repr()->lookup( + Cache::Handle* handle2 = DeleteBitmapAggCache::instance()->lookup( agg_cache_key(_tablet_id, {std::get<0>(bmk), std::get<1>(bmk), start_version})); DBUG_EXECUTE_IF("DeleteBitmap::get_agg.cache_miss", { @@ -1469,10 +1469,10 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co if (handle2 == nullptr || start_version > std::get<2>(bmk)) { start_version = 0; } else { - val->bitmap |= - reinterpret_cast(_agg_cache->repr()->value(handle2)) - ->bitmap; - _agg_cache->repr()->release(handle2); + val->bitmap |= reinterpret_cast( + DeleteBitmapAggCache::instance()->value(handle2)) + ->bitmap; + DeleteBitmapAggCache::instance()->release(handle2); VLOG_DEBUG << "get agg cache version=" << start_version << " for tablet=" << _tablet_id << ", rowset=" << std::get<0>(bmk).to_string() @@ -1492,8 +1492,9 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co val->bitmap |= bm; } } - size_t charge = val->bitmap.getSizeInBytes() + sizeof(AggCache::Value); - handle = _agg_cache->repr()->insert(key, val, charge, charge, CachePriority::NORMAL); + size_t charge = val->bitmap.getSizeInBytes() + sizeof(DeleteBitmapAggCache::Value); + handle = DeleteBitmapAggCache::instance()->insert(key, val, charge, charge, + CachePriority::NORMAL); if (config::enable_mow_get_agg_by_cache && !val->bitmap.isEmpty()) { std::lock_guard l(_rowset_cache_version_lock); // this version is already agg @@ -1520,7 +1521,7 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co // It is natural for the cache to reclaim the underlying memory return std::shared_ptr( - &val->bitmap, [this, handle](...) { _agg_cache->repr()->release(handle); }); + &val->bitmap, [handle](...) { DeleteBitmapAggCache::instance()->release(handle); }); } std::shared_ptr DeleteBitmap::get_agg_without_cache(const BitmapKey& bmk) const { @@ -1554,8 +1555,6 @@ DeleteBitmap DeleteBitmap::diffset(const std::set& key_set) const { return dbm; } -std::atomic DeleteBitmap::AggCache::s_repr {nullptr}; - std::string tablet_state_name(TabletState state) { switch (state) { case TABLET_NOTREADY: diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 63ad0124f862a6..949ce05fd74881 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -364,6 +364,20 @@ class TabletMeta : public MetadataAdder { mutable std::shared_mutex _meta_lock; }; +class DeleteBitmapAggCache : public LRUCachePolicy { +public: + DeleteBitmapAggCache(size_t capacity); + + static DeleteBitmapAggCache* instance(); + + static DeleteBitmapAggCache* create_instance(size_t capacity); + + class Value : public LRUCacheValueBase { + public: + roaring::Roaring bitmap; + }; +}; + /** * Wraps multiple bitmaps for recording rows (row id) that are deleted or * overwritten. For now, it's only used when unique key merge-on-write property @@ -578,40 +592,9 @@ class DeleteBitmap { */ DeleteBitmap diffset(const std::set& key_set) const; - class AggCachePolicy : public LRUCachePolicy { - public: - AggCachePolicy(size_t capacity) - : LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity, - LRUCacheType::SIZE, - config::delete_bitmap_agg_cache_stale_sweep_time_sec, 256) {} - }; - - class AggCache { - public: - class Value : public LRUCacheValueBase { - public: - roaring::Roaring bitmap; - }; - - AggCache(size_t size_in_bytes) { - static std::once_flag once; - std::call_once(once, [size_in_bytes] { - auto* tmp = new AggCachePolicy(size_in_bytes); - AggCache::s_repr.store(tmp, std::memory_order_release); - }); - - while (!s_repr.load(std::memory_order_acquire)) { - } - } - - static LRUCachePolicy* repr() { return s_repr.load(std::memory_order_acquire); } - static std::atomic s_repr; - }; - private: DeleteBitmap::Version _get_rowset_cache_version(const BitmapKey& bmk) const; - mutable std::shared_ptr _agg_cache; int64_t _tablet_id; mutable std::shared_mutex _rowset_cache_version_lock; mutable std::map> _rowset_cache_version; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index ec9fed17b7f85e..86a1e4c6c73066 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -115,6 +115,7 @@ class HeapProfiler; class WalManager; class DNSCache; struct SyncRowsetStats; +class DeleteBitmapAggCache; inline bool k_doris_exit = false; @@ -290,6 +291,7 @@ class ExecEnv { void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } + void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; } void set_tablet_column_object_pool(TabletColumnObjectPool* c) { this->_tablet_column_object_pool = c; } @@ -356,6 +358,8 @@ class ExecEnv { bool check_auth_token(const std::string& auth_token); + DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; } + private: ExecEnv(); @@ -485,6 +489,7 @@ class ExecEnv { QueryCache* _query_cache = nullptr; std::shared_ptr _dummy_lru_cache = nullptr; std::unique_ptr _file_cache_open_fd_cache; + DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr}; pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index fd6f61a294d8e2..8f30211602a30b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -54,6 +54,7 @@ #include "olap/segment_loader.h" #include "olap/storage_engine.h" #include "olap/tablet_column_object_pool.h" +#include "olap/tablet_meta.h" #include "olap/tablet_schema_cache.h" #include "olap/wal/wal_manager.h" #include "pipeline/pipeline_tracing.h" @@ -566,6 +567,16 @@ Status ExecEnv::_init_mem_env() { _query_cache = QueryCache::create_global_cache(config::query_cache_size * 1024L * 1024L); LOG(INFO) << "query cache memory limit: " << config::query_cache_size << "MB"; + // The default delete bitmap cache is set to 100MB, + // which can be insufficient and cause performance issues when the amount of user data is large. + // To mitigate the problem of an inadequate cache, + // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. + int64_t delete_bitmap_agg_cache_cache_limit = + ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit, + MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent); + _delete_bitmap_agg_cache = DeleteBitmapAggCache::create_instance(std::max( + delete_bitmap_agg_cache_cache_limit, config::delete_bitmap_agg_cache_capacity)); + return Status::OK(); } @@ -720,6 +731,7 @@ void ExecEnv::destroy() { SAFE_DELETE(_segment_loader); SAFE_DELETE(_row_cache); SAFE_DELETE(_query_cache); + SAFE_DELETE(_delete_bitmap_agg_cache); // Free resource after threads are stopped. // Some threads are still running, like threads created by _new_load_stream_mgr ... diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp index 810f3e0c28f548..253d17297804a6 100644 --- a/be/test/testutil/run_all_tests.cpp +++ b/be/test/testutil/run_all_tests.cpp @@ -32,6 +32,7 @@ #include "olap/segment_loader.h" #include "olap/storage_engine.h" #include "olap/tablet_column_object_pool.h" +#include "olap/tablet_meta.h" #include "olap/tablet_schema_cache.h" #include "runtime/exec_env.h" #include "runtime/memory/cache_manager.h" @@ -77,6 +78,9 @@ int main(int argc, char** argv) { doris::ExecEnv::GetInstance()->set_tablet_schema_cache( doris::TabletSchemaCache::create_global_schema_cache( doris::config::tablet_schema_cache_capacity)); + doris::ExecEnv::GetInstance()->set_delete_bitmap_agg_cache( + doris::DeleteBitmapAggCache::create_instance( + doris::config::delete_bitmap_agg_cache_capacity)); doris::ExecEnv::GetInstance()->set_tablet_column_object_pool( doris::TabletColumnObjectPool::create_global_column_cache( doris::config::tablet_schema_cache_capacity));