From d517208ea6d127eb59a6253f1084165942cc0ca0 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 20 Sep 2023 11:42:11 +0800 Subject: [PATCH] 2 --- be/src/common/config.cpp | 2 +- be/src/common/config.h | 3 +-- be/src/runtime/memory/mem_tracker_limiter.cpp | 2 +- be/src/runtime/memory/mem_tracker_limiter.h | 2 +- be/src/runtime/memory/thread_mem_tracker_mgr.h | 16 ++++++++++------ be/src/vec/common/allocator.cpp | 8 -------- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 74980aa41d53d1..c9c8ab3623c91b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -131,7 +131,7 @@ DEFINE_mBool(enable_query_memory_overcommit, "true"); DEFINE_mBool(disable_memory_gc, "false"); -DEFINE_mInt64(large_memory_check_bytes, "1073741824"); +DEFINE_mInt64(large_memory_check_bytes, "2147483648"); // The maximum time a thread waits for a full GC. Currently only query will wait for full gc. DEFINE_mInt32(thread_wait_gc_max_milliseconds, "1000"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 4c41e6f0e16423..ca88799ea90a5d 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -173,9 +173,8 @@ DECLARE_mBool(enable_query_memory_overcommit); // default gc strategy is conservative, if you want to exclude the interference of gc, let it be true DECLARE_mBool(disable_memory_gc); -// malloc or new large memory larger than large_memory_check_bytes and Doris Allocator is not used, +// malloc or new large memory larger than large_memory_check_bytes, default 2G, // will print a warning containing the stacktrace, but not prevent memory alloc. -// large memory alloc looking forward to using Allocator. DECLARE_mInt64(large_memory_check_bytes); // The maximum time a thread waits for a full GC. Currently only query will wait for full gc. diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index 10526efb820ea3..d82202569c0a93 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -256,7 +256,7 @@ std::string MemTrackerLimiter::log_process_usage_str() { std::vector snapshots; MemTrackerLimiter::make_process_snapshots(&snapshots); MemTrackerLimiter::make_type_snapshots(&snapshots, MemTrackerLimiter::Type::GLOBAL); - MemTrackerLimiter::make_top_consumption_snapshots(&snapshots); + MemTrackerLimiter::make_top_consumption_snapshots(&snapshots, 15); // Add additional tracker printed when memory exceeds limit. snapshots.emplace_back( diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h index 16df39c59ca5a2..41601494079a27 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.h +++ b/be/src/runtime/memory/mem_tracker_limiter.h @@ -150,7 +150,7 @@ class MemTrackerLimiter final : public MemTracker { static void make_process_snapshots(std::vector* snapshots); static void make_type_snapshots(std::vector* snapshots, Type type); static void make_top_consumption_snapshots(std::vector* snapshots, - int top_num = 15); + int top_num); static std::string log_usage(MemTracker::Snapshot snapshot); std::string log_usage() { return log_usage(make_snapshot()); } diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index e6fdfc80266802..e6ee0e3076ecae 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -33,6 +33,7 @@ #include "runtime/memory/mem_tracker.h" #include "runtime/memory/mem_tracker_limiter.h" #include "util/stack_util.h" +#include "util/uid_util.h" namespace doris { @@ -174,14 +175,17 @@ inline void ThreadMemTrackerMgr::consume(int64_t size, bool large_memory_check) !_stop_consume) { flush_untracked_mem(); } - // Large memory alloc should use allocator.h and catch std::bad_alloc - // Direct malloc or new large memory, unable to catch std::bad_alloc, BE may OOM. - if (large_memory_check && size > doris::config::large_memory_check_bytes) { + + if (large_memory_check && doris::config::large_memory_check_bytes <= 0 && + size > doris::config::large_memory_check_bytes) { _stop_consume = true; LOG(WARNING) << fmt::format( - "malloc or new large memory: {}, looking forward to using Allocator, this is just " - "a warning, not prevent memory alloc, stacktrace:\n{}", - size, get_stack_trace()); + "malloc or new large memory: {}, {}, this is just a warning, not prevent memory " + "alloc, stacktrace:\n{}", + size, + is_attach_query() ? "in query or load: " + print_id(_fragment_instance_id) + : "not in query or load", + get_stack_trace()); _stop_consume = false; } } diff --git a/be/src/vec/common/allocator.cpp b/be/src/vec/common/allocator.cpp index 4a168d82e67c8c..1f06e2f75c4c50 100644 --- a/be/src/vec/common/allocator.cpp +++ b/be/src/vec/common/allocator.cpp @@ -177,10 +177,6 @@ void Allocator::throw_bad_alloc( template void* Allocator::alloc(size_t size, size_t alignment) { - if (doris::enable_thread_catch_bad_alloc) { - doris::thread_context()->large_memory_check = false; - } - DEFER({ doris::thread_context()->large_memory_check = true; }); return alloc_impl(size, alignment); } @@ -188,10 +184,6 @@ template void* Allocator::realloc(void* buf, size_t old_size, size_t new_size, size_t alignment) { - if (doris::enable_thread_catch_bad_alloc) { - doris::thread_context()->large_memory_check = false; - } - DEFER({ doris::thread_context()->large_memory_check = true; }); return realloc_impl(buf, old_size, new_size, alignment); }