From 9ea7d225e6fd714409419e4bd8f627873c44db6f Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 11 Oct 2024 15:36:02 +0800 Subject: [PATCH] 2 --- be/src/http/default_path_handlers.cpp | 32 +++++----- be/src/runtime/memory/mem_tracker_limiter.cpp | 4 +- be/src/runtime/memory/memory_profile.cpp | 59 ++++++++++++------- be/src/runtime/memory/memory_profile.h | 18 +++++- be/src/runtime/process_profile.cpp | 3 +- be/src/runtime/process_profile.h | 12 ++++ 6 files changed, 86 insertions(+), 42 deletions(-) diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index f5eb30d2a209bc..e018fb04f0670d 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -93,24 +93,17 @@ void config_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* (*output) << ""; } -// Registered to handle "/profile". -void process_profile_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) { - (*output) << "

Process Profile

\n"; - doris::ProcessProfile::instance()->refresh_profile(); - (*output) << "
"
-              << doris::ProcessProfile::instance()->print_process_profile() << "
"; - +void memory_info_handler(std::stringstream* output) { (*output) << "

Memory Info

\n"; (*output) << "
";
-    (*output) << "
Memory Documents
\n"
+    (*output) << "
Memory Documents
\n"
               << "Memory Management Overview"
-              << std::endl
+                 "overview>Memory Management Overview\n"
               << "Memory Issue FAQ"
-              << std::endl;
+                 "memory-issue-faq>Memory Issue FAQ\n"
+              << "\n---\n";
 
     (*output) << "
Memory Properties
\n"
               << "System Physical Mem: "
@@ -128,7 +121,8 @@ void process_profile_handler(const WebPageHandler::ArgumentMap& args, std::strin
               << PrettyPrinter::print(MemInfo::cgroup_mem_limit(), TUnit::BYTES) << std::endl
               << "Cgroup Mem Usage: "
               << PrettyPrinter::print(MemInfo::cgroup_mem_usage(), TUnit::BYTES) << std::endl
-              << "Cgroup Mem Refresh State: " << MemInfo::cgroup_mem_refresh_state() << std::endl;
+              << "Cgroup Mem Refresh State: " << MemInfo::cgroup_mem_refresh_state() << std::endl
+              << "\n---\n";
 
     (*output) << "
Memory Option Settings
\n";
     {
@@ -141,6 +135,7 @@ void process_profile_handler(const WebPageHandler::ArgumentMap& args, std::strin
             }
         }
     }
+    (*output) << "\n---\n";
 
     (*output) << "
Jemalloc Profiles
\n";
 #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
@@ -165,6 +160,15 @@ void process_profile_handler(const WebPageHandler::ArgumentMap& args, std::strin
     (*output) << "
"; } +// Registered to handle "/profile". +void process_profile_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) { + (*output) << "

Process Profile

\n"; + doris::ProcessProfile::instance()->refresh_profile(); + (*output) << "
"
+              << doris::ProcessProfile::instance()->print_process_profile_no_root() << "
"; + memory_info_handler(output); +} + void display_tablets_callback(const WebPageHandler::ArgumentMap& args, EasyJson* ej) { std::string tablet_num_to_return; auto it = args.find("limit"); @@ -179,7 +183,7 @@ void display_tablets_callback(const WebPageHandler::ArgumentMap& args, EasyJson* // Registered to handle "/mem_tracker", and prints out memory tracker information. void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) { (*output) << "

mem_tracker webpage has been offline, please click Process Profile

\n"; + "href=../profile>Process Profile, see MemoryProfile and Memory Info\n"; } void heap_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) { diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index 2553ba2b2eec99..b8632852808088 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -39,7 +39,6 @@ namespace doris { static bvar::Adder memory_memtrackerlimiter_cnt("memory_memtrackerlimiter_cnt"); -std::atomic mem_tracker_number_counter(0); std::atomic mem_tracker_limiter_group_counter(0); constexpr auto GC_MAX_SEEK_TRACKER = 1000; @@ -62,8 +61,7 @@ static RuntimeProfile::Counter* previously_canceling_tasks_counter = MemTrackerLimiter::MemTrackerLimiter(Type type, const std::string& label, int64_t byte_limit) { DCHECK_GE(byte_limit, -1); _type = type; - // MemTrackerLimiter labels cannot be repeated, otherwise generation Runtime Profile will fail. - _label = fmt::format("{}_{}", label, mem_tracker_number_counter.fetch_add(1)); + _label = label; _limit = byte_limit; if (_type == Type::GLOBAL) { _group_num = 0; diff --git a/be/src/runtime/memory/memory_profile.cpp b/be/src/runtime/memory/memory_profile.cpp index 9643ead8793c7d..8dbdcbdd3af769 100644 --- a/be/src/runtime/memory/memory_profile.cpp +++ b/be/src/runtime/memory/memory_profile.cpp @@ -41,6 +41,8 @@ static bvar::Adder memory_untracked_memory_bytes("memory_untracked_memo MemoryProfile::MemoryProfile() { _memory_overview_profile.set(std::make_unique("MemoryOverviewSnapshot")); + _global_memory_profile.set(std::make_unique("GlobalMemorySnapshot")); + _top_memory_tasks_profile.set(std::make_unique("TopMemoryTasksSnapshot")); _tasks_memory_profile.set(std::make_unique("TasksMemorySnapshot")); } @@ -52,18 +54,16 @@ void MemoryProfile::refresh_memory_overview_profile() { std::unique_ptr memory_overview_profile = std::make_unique("MemoryOverviewSnapshot"); #endif + std::unique_ptr global_memory_profile = + std::make_unique("GlobalMemorySnapshot"); + std::unique_ptr top_memory_tasks_profile = + std::make_unique("TopMemoryTasksSnapshot"); // 1. create profile - RuntimeProfile* basic_overview_profile = - memory_overview_profile->create_child("BasicOverview", true, false); - RuntimeProfile* top_memory_tasks_profile = - memory_overview_profile->create_child("TopMemoryTasks", true, false); - RuntimeProfile* global_memory_profile = - memory_overview_profile->create_child("GlobalMemory", true, false); RuntimeProfile* untracked_memory_profile = - basic_overview_profile->create_child("UntrackedMemory", true, false); + memory_overview_profile->create_child("UntrackedMemory", true, false); RuntimeProfile* tracked_memory_profile = - basic_overview_profile->create_child("TrackedMemory", true, false); + memory_overview_profile->create_child("TrackedMemory", true, false); RuntimeProfile* tasks_memory_overview_profile = tracked_memory_profile->create_child("TasksMemory", true, false); RuntimeProfile* tasks_memory_overview_details_profile = @@ -78,13 +78,13 @@ void MemoryProfile::refresh_memory_overview_profile() { // 2. add counter // 2.1 add process memory counter RuntimeProfile::Counter* process_physical_memory_current_usage_counter = - ADD_COUNTER(basic_overview_profile, "PhysicalMemory(VmRSS)", TUnit::BYTES); + ADD_COUNTER(memory_overview_profile, "PhysicalMemory(VmRSS)", TUnit::BYTES); RuntimeProfile::Counter* process_physical_memory_peak_usage_counter = - basic_overview_profile->AddHighWaterMarkCounter("PhysicalMemoryPeak", TUnit::BYTES); + memory_overview_profile->AddHighWaterMarkCounter("PhysicalMemoryPeak", TUnit::BYTES); RuntimeProfile::Counter* process_virtual_memory_current_usage_counter = - ADD_COUNTER(basic_overview_profile, "VirtualMemory(VmSize)", TUnit::BYTES); + ADD_COUNTER(memory_overview_profile, "VirtualMemory(VmSize)", TUnit::BYTES); RuntimeProfile::Counter* process_virtual_memory_peak_usage_counter = - basic_overview_profile->AddHighWaterMarkCounter("VirtualMemoryPeak", TUnit::BYTES); + memory_overview_profile->AddHighWaterMarkCounter("VirtualMemoryPeak", TUnit::BYTES); // 2.2 add untracked memory counter RuntimeProfile::Counter* untracked_memory_current_usage_counter = @@ -233,10 +233,11 @@ void MemoryProfile::refresh_memory_overview_profile() { } } - MemTrackerLimiter::make_type_trackers_profile(global_memory_profile, + MemTrackerLimiter::make_type_trackers_profile(global_memory_profile.get(), MemTrackerLimiter::Type::GLOBAL); - MemTrackerLimiter::make_top_consumption_tasks_tracker_profile(top_memory_tasks_profile, 15); + MemTrackerLimiter::make_top_consumption_tasks_tracker_profile(top_memory_tasks_profile.get(), + 15); COUNTER_SET(tasks_memory_current_usage_counter, tasks_trackers_mem_sum); COUNTER_SET(tasks_memory_peak_usage_counter, tasks_trackers_mem_sum); @@ -287,6 +288,8 @@ void MemoryProfile::refresh_memory_overview_profile() { // 4. reset profile _memory_overview_profile.set(std::move(memory_overview_profile)); + _global_memory_profile.set(std::move(global_memory_profile)); + _top_memory_tasks_profile.set(std::move(top_memory_tasks_profile)); } void MemoryProfile::refresh_tasks_memory_profile() { @@ -297,15 +300,27 @@ void MemoryProfile::refresh_tasks_memory_profile() { } void MemoryProfile::make_memory_profile(RuntimeProfile* profile) const { - RuntimeProfile* memory_profile_snapshot = - profile->create_child("MemoryProfileSnapshot", true, false); + RuntimeProfile* memory_profile_snapshot = profile->create_child("MemoryProfile", true, false); + auto memory_overview_version_ptr = _memory_overview_profile.get(); - auto tasks_memory_version_ptr = _tasks_memory_profile.get(); RuntimeProfile* memory_overview_profile = memory_profile_snapshot->create_child(memory_overview_version_ptr->name(), true, false); + memory_overview_profile->merge(const_cast(memory_overview_version_ptr.get())); + + auto global_memory_version_ptr = _global_memory_profile.get(); + RuntimeProfile* global_memory_profile = + memory_profile_snapshot->create_child(global_memory_version_ptr->name(), true, false); + global_memory_profile->merge(const_cast(global_memory_version_ptr.get())); + + auto top_memory_tasks_version_ptr = _top_memory_tasks_profile.get(); + RuntimeProfile* top_memory_tasks_profile = memory_profile_snapshot->create_child( + top_memory_tasks_version_ptr->name(), true, false); + top_memory_tasks_profile->merge( + const_cast(top_memory_tasks_version_ptr.get())); + + auto tasks_memory_version_ptr = _tasks_memory_profile.get(); RuntimeProfile* tasks_memory_profile = memory_profile_snapshot->create_child(tasks_memory_version_ptr->name(), true, false); - memory_overview_profile->merge(const_cast(memory_overview_version_ptr.get())); tasks_memory_profile->merge(const_cast(tasks_memory_version_ptr.get())); } @@ -328,10 +343,10 @@ int64_t MemoryProfile::other_current_usage() { void MemoryProfile::print_log_process_usage() { if (_enable_print_log_process_usage) { _enable_print_log_process_usage = false; - std::string detail; - detail += "\nProcess Memory Summary: " + GlobalMemoryArbitrator::process_mem_log_str(); - detail += "\n" + print_memory_overview_profile(); - LOG(WARNING) << detail; + LOG(WARNING) << "Process Memory Summary: " + GlobalMemoryArbitrator::process_mem_log_str(); + LOG(WARNING) << "\n" << print_memory_overview_profile(); + LOG(WARNING) << "\n" << print_global_memory_profile(); + LOG(WARNING) << "\n" << print_top_memory_tasks_profile(); } } diff --git a/be/src/runtime/memory/memory_profile.h b/be/src/runtime/memory/memory_profile.h index 4b3a8d05cadc3d..9f1bab0c02a802 100644 --- a/be/src/runtime/memory/memory_profile.h +++ b/be/src/runtime/memory/memory_profile.h @@ -33,15 +33,29 @@ class MemoryProfile { void make_memory_profile(RuntimeProfile* profile) const; std::string print_memory_overview_profile() const { + std::stringstream ss; auto version_ptr = _memory_overview_profile.get(); + version_ptr->pretty_print(&ss); + return ss.str(); + } + + std::string print_global_memory_profile() const { + std::stringstream ss; + auto version_ptr = _global_memory_profile.get(); + version_ptr->pretty_print(&ss); + return ss.str(); + } + + std::string print_top_memory_tasks_profile() const { std::stringstream ss; + auto version_ptr = _top_memory_tasks_profile.get(); version_ptr->pretty_print(&ss); return ss.str(); } std::string print_tasks_memory_profile() const { - auto version_ptr = _tasks_memory_profile.get(); std::stringstream ss; + auto version_ptr = _tasks_memory_profile.get(); version_ptr->pretty_print(&ss); return ss.str(); } @@ -58,6 +72,8 @@ class MemoryProfile { private: MultiVersion _memory_overview_profile; + MultiVersion _global_memory_profile; + MultiVersion _top_memory_tasks_profile; MultiVersion _tasks_memory_profile; std::atomic _enable_print_log_process_usage {true}; diff --git a/be/src/runtime/process_profile.cpp b/be/src/runtime/process_profile.cpp index 24327b63891e9a..d91aedbeac2025 100644 --- a/be/src/runtime/process_profile.cpp +++ b/be/src/runtime/process_profile.cpp @@ -24,7 +24,6 @@ namespace doris { ProcessProfile::ProcessProfile() { - _process_profile.set(std::make_unique("ProcessProfile")); _memory_profile = std::make_unique(); } @@ -38,8 +37,8 @@ void ProcessProfile::refresh_profile() { std::unique_ptr process_profile = std::make_unique("ProcessProfile"); _memory_profile->make_memory_profile(process_profile.get()); - // TODO make other profile _process_profile.set(std::move(process_profile)); + // TODO make other profile } } // namespace doris diff --git a/be/src/runtime/process_profile.h b/be/src/runtime/process_profile.h index 64653bb566b115..24b128ab5528e2 100644 --- a/be/src/runtime/process_profile.h +++ b/be/src/runtime/process_profile.h @@ -40,6 +40,18 @@ class ProcessProfile { return ss.str(); } + std::string print_process_profile_no_root() const { + std::stringstream ss; + std::vector profiles; + auto version_ptr = _process_profile.get(); + auto* process_profile = const_cast(version_ptr.get()); + process_profile->get_children(&profiles); + for (auto* profile : profiles) { + profile->pretty_print(&ss); + } + return ss.str(); + } + MemoryProfile* memory_profile() { return _memory_profile.get(); } private: