diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index a6d504f84d60b3..44309a66153ab9 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -118,6 +118,8 @@ DEFINE_mInt32(double_resize_threshold, "23"); DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "6871947673"); +DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800"); + // The size of the memory that gc wants to release each time, as a percentage of the mem limit. DEFINE_mString(process_minor_gc_size, "10%"); DEFINE_mString(process_full_gc_size, "20%"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 6b90bc87dde242..d3f2a439b60440 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -165,6 +165,9 @@ DECLARE_mInt32(double_resize_threshold); // Turn down max. will use as much memory as possible. DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes); +// reserve a small amount of memory so we do not trigger MinorGC +DECLARE_Int64(memtable_limiter_reserved_memory_bytes); + // The size of the memory that gc wants to release each time, as a percentage of the mem limit. DECLARE_mString(process_minor_gc_size); DECLARE_mString(process_full_gc_size); diff --git a/be/src/olap/memtable_memory_limiter.cpp b/be/src/olap/memtable_memory_limiter.cpp index 1eaad31ec22724..23b760284b8985 100644 --- a/be/src/olap/memtable_memory_limiter.cpp +++ b/be/src/olap/memtable_memory_limiter.cpp @@ -77,20 +77,17 @@ void MemTableMemoryLimiter::register_writer(std::weak_ptr writer _writers.push_back(writer); } -int64_t MemTableMemoryLimiter::_avail_mem_lack() { +bool MemTableMemoryLimiter::_sys_avail_mem_less_than_warning_water_mark() { // reserve a small amount of memory so we do not trigger MinorGC - auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark(); - auto avail_mem_lack = doris::MemInfo::sys_mem_available_warning_water_mark() - - doris::GlobalMemoryArbitrator::sys_mem_available(); - return avail_mem_lack + reserved_mem; + return doris::GlobalMemoryArbitrator::sys_mem_available() < + doris::MemInfo::sys_mem_available_warning_water_mark() + + config::memtable_limiter_reserved_memory_bytes; } -int64_t MemTableMemoryLimiter::_proc_mem_extra() { +bool MemTableMemoryLimiter::_process_used_mem_more_than_soft_mem_limit() { // reserve a small amount of memory so we do not trigger MinorGC - auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark(); - auto proc_mem_extra = - GlobalMemoryArbitrator::process_memory_usage() - MemInfo::soft_mem_limit(); - return proc_mem_extra + reserved_mem; + return GlobalMemoryArbitrator::process_memory_usage() > + MemInfo::soft_mem_limit() - config::memtable_limiter_reserved_memory_bytes; } bool MemTableMemoryLimiter::_soft_limit_reached() { @@ -98,8 +95,9 @@ bool MemTableMemoryLimiter::_soft_limit_reached() { } bool MemTableMemoryLimiter::_hard_limit_reached() { - return _mem_tracker->consumption() >= _load_hard_mem_limit || _avail_mem_lack() >= 0 || - _proc_mem_extra() >= 0; + return _mem_tracker->consumption() >= _load_hard_mem_limit || + _sys_avail_mem_less_than_warning_water_mark() || + _process_used_mem_more_than_soft_mem_limit(); } bool MemTableMemoryLimiter::_load_usage_low() { diff --git a/be/src/olap/memtable_memory_limiter.h b/be/src/olap/memtable_memory_limiter.h index 38cd83e61dcd8d..2e8271bab35c15 100644 --- a/be/src/olap/memtable_memory_limiter.h +++ b/be/src/olap/memtable_memory_limiter.h @@ -51,8 +51,8 @@ class MemTableMemoryLimiter { int64_t mem_usage() const { return _mem_usage; } private: - static int64_t _avail_mem_lack(); - static int64_t _proc_mem_extra(); + static inline bool _sys_avail_mem_less_than_warning_water_mark(); + static inline bool _process_used_mem_more_than_soft_mem_limit(); bool _soft_limit_reached(); bool _hard_limit_reached();