[opt](memory) Modify thedefault JEMALLOC_CONF and support flush Jemal…

…loc tcache (apache#38185)

1. Modify thedefault JEMALLOC_CONF, reduce `muzzy_decay_ms` and
`dirty_decay_ms`, and modify the parameters of the regular dump heap
profile for better understanding.
2. support flush Jemalloc tcache, when process memory exceed limit and
tcache is larger than 1G, the limit of `tcache` is the number of pages,
not the total number of page bytes.

.licenserc.yaml

@@ -93,4 +93,5 @@ header:
     - "pytest/sys/data"
     - "pytest/deploy/*.conf"
     - "tools/FlameGraph/*"
+    - "tools/jeprof"
   comment: on-failure

be/src/util/mem_info.h

@@ -146,15 +146,34 @@ class MemInfo {
         if (config::enable_je_purge_dirty_pages) {
             try {
                 // Purge all unused dirty pages for arena <i>, or for all arenas if <i> equals MALLCTL_ARENAS_ALL.
-                jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr,
-                          nullptr, nullptr, 0);
+                int err = jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(),
+                                    nullptr, nullptr, nullptr, 0);
+                if (err) {
+                    LOG(WARNING) << "Jemalloc purge all unused dirty pages failed";
+                }
             } catch (...) {
                 LOG(WARNING) << "Purge all unused dirty pages for all arenas failed";
             }
         }
 #endif
     }
 
+    // the limit of `tcache` is the number of pages, not the total number of page bytes.
+    // `tcache` has two cleaning opportunities: 1. the number of memory alloc and releases reaches a certain number,
+    // recycle pages that has not been used for a long time; 2. recycle all `tcache` when the thread exits.
+    // here add a total size limit.
+    static inline void je_thread_tcache_flush() {
+#ifdef USE_JEMALLOC
+        constexpr size_t TCACHE_LIMIT = (1ULL << 30); // 1G
+        if (allocator_cache_mem() - je_dirty_pages_mem() > TCACHE_LIMIT) {
+            int err = jemallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
+            if (err) {
+                LOG(WARNING) << "Jemalloc thread.tcache.flush failed";
+            }
+        }
+#endif
+    }
+
     static std::mutex je_purge_dirty_pages_lock;
     static std::condition_variable je_purge_dirty_pages_cv;
     static std::atomic<bool> je_purge_dirty_pages_notify;

be/src/vec/common/allocator.cpp

@@ -112,6 +112,8 @@ void Allocator<clear_memory_, mmap_populate, use_mmap, MemoryAllocator>::sys_mem
                     print_id(doris::thread_context()->task_id()),
                     doris::thread_context()->get_thread_id(),
                     doris::config::thread_wait_gc_max_milliseconds, err_msg);
+            // only query thread exceeded memory limit for the first time and wait_gc is true.
+            doris::MemInfo::je_thread_tcache_flush();
             if (!doris::config::disable_memory_gc) {
                 while (wait_milliseconds < doris::config::thread_wait_gc_max_milliseconds) {
                     std::this_thread::sleep_for(std::chrono::milliseconds(100));