Skip to content

Commit

Permalink
[opt](memory) Modify thedefault JEMALLOC_CONF and support flush Jemal…
Browse files Browse the repository at this point in the history
…loc tcache (apache#38185)

1. Modify thedefault JEMALLOC_CONF, reduce `muzzy_decay_ms` and
`dirty_decay_ms`, and modify the parameters of the regular dump heap
profile for better understanding.
2. support flush Jemalloc tcache, when process memory exceed limit and
tcache is larger than 1G, the limit of `tcache` is the number of pages,
not the total number of page bytes.
  • Loading branch information
xinyiZzz authored Aug 21, 2024
1 parent ca9a294 commit 81160b2
Show file tree
Hide file tree
Showing 11 changed files with 5,758 additions and 13 deletions.
1 change: 1 addition & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,5 @@ header:
- "pytest/qe"
- "pytest/sys/data"
- "pytest/deploy/*.conf"
- "tools/jeprof"
comment: on-failure
23 changes: 21 additions & 2 deletions be/src/util/mem_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -146,15 +146,34 @@ class MemInfo {
if (config::enable_je_purge_dirty_pages) {
try {
// Purge all unused dirty pages for arena <i>, or for all arenas if <i> equals MALLCTL_ARENAS_ALL.
jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr,
nullptr, nullptr, 0);
int err = jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(),
nullptr, nullptr, nullptr, 0);
if (err) {
LOG(WARNING) << "Jemalloc purge all unused dirty pages failed";
}
} catch (...) {
LOG(WARNING) << "Purge all unused dirty pages for all arenas failed";
}
}
#endif
}

// the limit of `tcache` is the number of pages, not the total number of page bytes.
// `tcache` has two cleaning opportunities: 1. the number of memory alloc and releases reaches a certain number,
// recycle pages that has not been used for a long time; 2. recycle all `tcache` when the thread exits.
// here add a total size limit.
static inline void je_thread_tcache_flush() {
#ifdef USE_JEMALLOC
constexpr size_t TCACHE_LIMIT = (1ULL << 30); // 1G
if (allocator_cache_mem() - je_dirty_pages_mem() > TCACHE_LIMIT) {
int err = jemallctl("thread.tcache.flush", nullptr, nullptr, nullptr, 0);
if (err) {
LOG(WARNING) << "Jemalloc thread.tcache.flush failed";
}
}
#endif
}

static std::mutex je_purge_dirty_pages_lock;
static std::condition_variable je_purge_dirty_pages_cv;
static std::atomic<bool> je_purge_dirty_pages_notify;
Expand Down
2 changes: 2 additions & 0 deletions be/src/vec/common/allocator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ void Allocator<clear_memory_, mmap_populate, use_mmap, MemoryAllocator>::sys_mem
print_id(doris::thread_context()->task_id()),
doris::thread_context()->get_thread_id(),
doris::config::thread_wait_gc_max_milliseconds, err_msg);
// only query thread exceeded memory limit for the first time and wait_gc is true.
doris::MemInfo::je_thread_tcache_flush();
if (!doris::config::disable_memory_gc) {
while (wait_milliseconds < doris::config::thread_wait_gc_max_milliseconds) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
Expand Down
2 changes: 1 addition & 1 deletion bin/run-fs-benchmark.sh
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ export LIBHDFS_OPTS="${final_java_opt}"
#echo "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"

# see https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"
export AWS_EC2_METADATA_DISABLED=true
export AWS_MAX_ATTEMPTS=2

Expand Down
2 changes: 1 addition & 1 deletion bin/start_be.sh
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ export LIBHDFS_OPTS="${final_java_opt}"
# log "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"

if [[ -z ${JEMALLOC_CONF} ]]; then
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"
fi

if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then
Expand Down
2 changes: 1 addition & 1 deletion cloud/script/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ fi

echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}"

export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:true,prof_prefix:jeprof.out"
export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"

if [[ "${RUN_VERSION}" -eq 1 ]]; then
"${bin}" --version
Expand Down
4 changes: 2 additions & 2 deletions conf/be.conf
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$LOG_DIR/jni.log -Xlog:gc*:$LOG_DIR/be

# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
# https://jemalloc.net/jemalloc.3.html
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_PROF_PRFIX=""
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"
JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_"

# ports for admin, web, heartbeat service
be_port = 9060
Expand Down
4 changes: 2 additions & 2 deletions regression-test/pipeline/external/conf/be.conf
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DOR

# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
# https://jemalloc.net/jemalloc.3.html
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_PROF_PRFIX=""
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"
JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_"

# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
Expand Down
4 changes: 2 additions & 2 deletions regression-test/pipeline/p0/conf/be.conf
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DOR

# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
# https://jemalloc.net/jemalloc.3.html
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_PROF_PRFIX=""
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"
JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_"

# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
Expand Down
4 changes: 2 additions & 2 deletions regression-test/pipeline/p1/conf/be.conf
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DOR

# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
# https://jemalloc.net/jemalloc.3.html
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
JEMALLOC_PROF_PRFIX=""
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:false,lg_prof_interval:-1"
JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_"

# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
Expand Down
Loading

0 comments on commit 81160b2

Please sign in to comment.