1

xinyiZzz · Jul 2, 2024 · 4ab0248 · 4ab0248
1 parent 80f6583
commit 4ab0248
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 0 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -566,6 +566,7 @@ DEFINE_String(pprof_profile_dir, "${DORIS_HOME}/log");
 // for jeprofile in jemalloc
 DEFINE_mString(jeprofile_dir, "${DORIS_HOME}/log");
 DEFINE_mBool(enable_je_purge_dirty_pages, "true");
+DEFINE_mString(je_dirty_pages_mem_limit_percent, "5%");
 
 // to forward compatibility, will be removed later
 DEFINE_mBool(enable_token_check, "true");

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -617,6 +617,8 @@ DECLARE_String(pprof_profile_dir);
 DECLARE_mString(jeprofile_dir);
 // Purge all unused dirty pages for all arenas.
 DECLARE_mBool(enable_je_purge_dirty_pages);
+// Purge all unused Jemalloc dirty pages for all arenas when exceed limit percent * mem_limit.
+DECLARE_mString(je_dirty_pages_mem_limit_percent);
 
 // to forward compatibility, will be removed later
 DECLARE_mBool(enable_token_check);

diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
@@ -225,6 +225,11 @@ void Daemon::memory_maintenance_thread() {
             // Refresh allocator memory metrics.
 #if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && !defined(THREAD_SANITIZER)
             doris::MemInfo::refresh_allocator_mem();
+#ifdef USE_JEMALLOC
+            if (doris::MemInfo::je_dirty_pages_mem() > doris::MemInfo::je_dirty_pages_mem_limit()) {
+                doris::MemInfo::notify_je_purge_dirty_pages();
+            }
+#endif
             if (config::enable_system_metrics) {
                 DorisMetrics::instance()->system_metrics()->update_allocator_metrics();
             }

diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
@@ -52,6 +52,8 @@ std::atomic<int64_t> MemInfo::_s_mem_limit = std::numeric_limits<int64_t>::max()
 std::atomic<int64_t> MemInfo::_s_soft_mem_limit = std::numeric_limits<int64_t>::max();
 
 std::atomic<int64_t> MemInfo::_s_allocator_cache_mem = 0;
+std::atomic<int64_t> MemInfo::_s_je_dirty_pages_mem = std::numeric_limits<int64_t>::min();
+std::atomic<int64_t> MemInfo::_s_je_dirty_pages_mem_limit = std::numeric_limits<int64_t>::max();
 std::atomic<int64_t> MemInfo::_s_virtual_memory_used = 0;
 
 int64_t MemInfo::_s_cgroup_mem_limit = std::numeric_limits<int64_t>::max();
@@ -86,6 +88,8 @@ void MemInfo::refresh_allocator_mem() {
                                          get_je_metrics("stats.metadata") +
                                          get_je_all_arena_metrics("pdirty") * get_page_size(),
                                  std::memory_order_relaxed);
+    _s_je_dirty_pages_mem.store(get_je_all_arena_metrics("pdirty") * get_page_size(),
+                                std::memory_order_relaxed);
     _s_virtual_memory_used.store(get_je_metrics("stats.mapped"), std::memory_order_relaxed);
 #else
     _s_allocator_cache_mem.store(get_tc_metrics("tcmalloc.pageheap_free_bytes") +
@@ -244,6 +248,8 @@ void MemInfo::refresh_proc_meminfo() {
                                                                  _s_mem_limit, &is_percent));
         _s_process_full_gc_size.store(ParseUtil::parse_mem_spec(config::process_full_gc_size, -1,
                                                                 _s_mem_limit, &is_percent));
+        _s_je_dirty_pages_mem_limit.store(ParseUtil::parse_mem_spec(
+                config::je_dirty_pages_mem_limit_percent, -1, _s_mem_limit, &is_percent));
     }
 
     // 3. refresh process available memory

diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
@@ -147,6 +147,12 @@ class MemInfo {
     static inline size_t allocator_cache_mem() {
         return _s_allocator_cache_mem.load(std::memory_order_relaxed);
     }
+    static inline int64_t je_dirty_pages_mem() {
+        return _s_je_dirty_pages_mem.load(std::memory_order_relaxed);
+    }
+    static inline int64_t je_dirty_pages_mem_limit() {
+        return _s_je_dirty_pages_mem_limit.load(std::memory_order_relaxed);
+    }
 
     // Tcmalloc property `generic.total_physical_bytes` records the total length of the virtual memory
     // obtained by the process malloc, not the physical memory actually used by the process in the OS.
@@ -181,6 +187,8 @@ class MemInfo {
     static std::atomic<int64_t> _s_soft_mem_limit;
 
     static std::atomic<int64_t> _s_allocator_cache_mem;
+    static std::atomic<int64_t> _s_je_dirty_pages_mem;
+    static std::atomic<int64_t> _s_je_dirty_pages_mem_limit;
     static std::atomic<int64_t> _s_virtual_memory_used;
 
     static int64_t _s_cgroup_mem_limit;