diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 92303473ad6f92..412c32e28a29a6 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -566,6 +566,7 @@ DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log"); // for jeprofile in jemalloc DEFINE_mString(jeprofile_dir, "${DORIS_HOME}/log"); DEFINE_mBool(enable_je_purge_dirty_pages, "true"); +DEFINE_mString(je_dirty_pages_mem_limit_percent, "5%"); // to forward compatibility, will be removed later DEFINE_mBool(enable_token_check, "true"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 1a9e3291db5adb..bae42dc0f96adc 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -617,6 +617,8 @@ DECLARE_String(pprof_profile_dir); DECLARE_mString(jeprofile_dir); // Purge all unused dirty pages for all arenas. DECLARE_mBool(enable_je_purge_dirty_pages); +// Purge all unused Jemalloc dirty pages for all arenas when exceed limit percent * mem_limit. +DECLARE_mString(je_dirty_pages_mem_limit_percent); // to forward compatibility, will be removed later DECLARE_mBool(enable_token_check); diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 44f21c47e79a69..dd14d5c723281f 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -225,6 +225,11 @@ void Daemon::memory_maintenance_thread() { // Refresh allocator memory metrics. #if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER) doris::MemInfo::refresh_allocator_mem(); +#ifdef USE_JEMALLOC + if (doris::MemInfo::je_dirty_pages_mem() > doris::MemInfo::je_dirty_pages_mem_limit()) { + doris::MemInfo::notify_je_purge_dirty_pages(); + } +#endif if (config::enable_system_metrics) { DorisMetrics::instance()->system_metrics()->update_allocator_metrics(); } diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index 45e609d7100431..57e48e947ffb46 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -52,6 +52,8 @@ std::atomic MemInfo::_s_mem_limit = std::numeric_limits::max() std::atomic MemInfo::_s_soft_mem_limit = std::numeric_limits::max(); std::atomic MemInfo::_s_allocator_cache_mem = 0; +std::atomic MemInfo::_s_je_dirty_pages_mem = std::numeric_limits::min(); +std::atomic MemInfo::_s_je_dirty_pages_mem_limit = std::numeric_limits::max(); std::atomic MemInfo::_s_virtual_memory_used = 0; int64_t MemInfo::_s_cgroup_mem_limit = std::numeric_limits::max(); @@ -86,6 +88,8 @@ void MemInfo::refresh_allocator_mem() { get_je_metrics("stats.metadata") + get_je_all_arena_metrics("pdirty") * get_page_size(), std::memory_order_relaxed); + _s_je_dirty_pages_mem.store(get_je_all_arena_metrics("pdirty") * get_page_size(), + std::memory_order_relaxed); _s_virtual_memory_used.store(get_je_metrics("stats.mapped"), std::memory_order_relaxed); #else _s_allocator_cache_mem.store(get_tc_metrics("tcmalloc.pageheap_free_bytes") + @@ -244,6 +248,8 @@ void MemInfo::refresh_proc_meminfo() { _s_mem_limit, &is_percent)); _s_process_full_gc_size.store(ParseUtil::parse_mem_spec(config::process_full_gc_size, -1, _s_mem_limit, &is_percent)); + _s_je_dirty_pages_mem_limit.store(ParseUtil::parse_mem_spec( + config::je_dirty_pages_mem_limit_percent, -1, _s_mem_limit, &is_percent)); } // 3. refresh process available memory diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index 28e5e05b5fa30b..f5673c1a166f02 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -147,6 +147,12 @@ class MemInfo { static inline size_t allocator_cache_mem() { return _s_allocator_cache_mem.load(std::memory_order_relaxed); } + static inline int64_t je_dirty_pages_mem() { + return _s_je_dirty_pages_mem.load(std::memory_order_relaxed); + } + static inline int64_t je_dirty_pages_mem_limit() { + return _s_je_dirty_pages_mem_limit.load(std::memory_order_relaxed); + } // Tcmalloc property `generic.total_physical_bytes` records the total length of the virtual memory // obtained by the process malloc, not the physical memory actually used by the process in the OS. @@ -181,6 +187,8 @@ class MemInfo { static std::atomic _s_soft_mem_limit; static std::atomic _s_allocator_cache_mem; + static std::atomic _s_je_dirty_pages_mem; + static std::atomic _s_je_dirty_pages_mem_limit; static std::atomic _s_virtual_memory_used; static int64_t _s_cgroup_mem_limit;