[fix](stack trace) Optimize stack trace output (apache#24933)

Status prints the stack trace, the first four frame pointers are removed, it doesn't make sense.
Optimize stack trace field order.
example:

  0#  doris::PlanFragmentExecutor::cancel(doris::PPlanFragmentCancelReason const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at doris/core/be/src/common/status.h:0
  1#  doris::FragmentMgr::cancel_query_unlocked(doris::TUniqueId const&, doris::PPlanFragmentCancelReason const&, std::unique_lock<std::mutex> const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at doris/cor
e/be/src/runtime/fragment_mgr.cpp:984
  2#  doris::FragmentMgr::cancel_query(doris::TUniqueId const&, doris::PPlanFragmentCancelReason const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) at doris/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/
../../../../include/x86_64-linux-gnu/c++/11/bits/gthr-default.h:778
  3#  long doris::MemTrackerLimiter::free_top_memory_query<doris::MemTrackerLimiter::TrackerLimiterGroup>(long, doris::MemTrackerLimiter::Type, std::vector<doris::MemTrackerLimiter::TrackerLimiterGroup, std::allocator<doris::MemTrackerLimiter::TrackerLimiterGroup> >&, s
td::function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > (long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)> const&, doris::RuntimeProfile*) at doris/ldb_toolchain/bin/../
lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187
  4#  doris::MemTrackerLimiter::free_top_memory_query(long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, doris::RuntimeProfile*, doris::Mem
TrackerLimiter::Type) at doris/core/be/src/runtime/memory/mem_tracker_limiter.cpp:362
  5#  doris::MemInfo::process_full_gc() at doris/core/be/src/util/mem_info.cpp:198
  6#  doris::Daemon::memory_gc_thread() at doris/core/be/src/common/daemon.cpp:0
  7#  doris::Thread::supervise_thread(void*) at doris/ldb_toolchain/bin/../usr/include/pthread.h:562
  8#  start_thread
  9#  __clone

be/src/common/stack_trace.cpp

@@ -49,14 +49,14 @@ namespace {
 /// But we use atomic just in case, so it is possible to be modified at runtime.
 std::atomic<bool> show_addresses = true;
 
-#if defined(__ELF__) && !defined(__FreeBSD__)
-void writePointerHex(const void* ptr, std::stringstream& buf) {
-    buf.write("0x", 2);
-    char hex_str[2 * sizeof(ptr)];
-    doris::vectorized::write_hex_uint_lowercase(reinterpret_cast<uintptr_t>(ptr), hex_str);
-    buf.write(hex_str, 2 * sizeof(ptr));
-}
-#endif
+// #if defined(__ELF__) && !defined(__FreeBSD__)
+// void writePointerHex(const void* ptr, std::stringstream& buf) {
+//     buf.write("0x", 2);
+//     char hex_str[2 * sizeof(ptr)];
+//     doris::vectorized::write_hex_uint_lowercase(reinterpret_cast<uintptr_t>(ptr), hex_str);
+//     buf.write(hex_str, 2 * sizeof(ptr));
+// }
+// #endif
 
 bool shouldShowAddress(const void* addr) {
     /// If the address is less than 4096, most likely it is a nullptr dereference with offset,
@@ -380,20 +380,15 @@ static void toStringEveryLineImpl([[maybe_unused]] const std::string dwarf_locat
                 reinterpret_cast<const void*>(uintptr_t(virtual_addr) - virtual_offset);
 
         std::stringstream out;
-        out << "\t" << i << ". ";
+        out << "\t" << i << "# ";
         if (i < 10) { // for alignment
             out << " ";
         }
 
-        if (shouldShowAddress(physical_addr)) {
-            out << "@ ";
-            writePointerHex(physical_addr, out);
-        }
-
         if (const auto* const symbol = symbol_index.findSymbol(virtual_addr)) {
-            out << "  " << collapseNames(demangle(symbol->name));
+            out << collapseNames(demangle(symbol->name));
         } else {
-            out << " ?";
+            out << "?";
         }
 
         if (std::error_code ec; object && std::filesystem::exists(object->name, ec) && !ec) {
@@ -403,11 +398,17 @@ static void toStringEveryLineImpl([[maybe_unused]] const std::string dwarf_locat
 
             if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, mode,
                                              inline_frames)) {
-                out << "  " << location.file.toString() << ":" << location.line;
+                out << " at " << location.file.toString() << ":" << location.line;
             }
         }
 
-        out << "  in " << (object ? object->name : "?");
+        // Do not display the stack address and file name, it is not important.
+        // if (shouldShowAddress(physical_addr)) {
+        //     out << " @ ";
+        //     writePointerHex(physical_addr, out);
+        // }
+
+        // out << "  in " << (object ? object->name : "?");
 
         callback(out.str());
 
@@ -458,11 +459,13 @@ std::string toStringCached(const StackTrace::FramePointers& pointers, size_t off
     }
 }
 
-std::string StackTrace::toString() const {
-    // Delete the first three frame pointers, which are inside the stacktrace.
+std::string StackTrace::toString(int start_pointers_index) const {
+    // Default delete the first three frame pointers, which are inside the stack_trace.cpp.
+    start_pointers_index += 3;
     StackTrace::FramePointers frame_pointers_raw {};
-    std::copy(frame_pointers.begin() + 3, frame_pointers.end(), frame_pointers_raw.begin());
-    return toStringCached(frame_pointers_raw, offset, size - 3);
+    std::copy(frame_pointers.begin() + start_pointers_index, frame_pointers.end(),
+              frame_pointers_raw.begin());
+    return toStringCached(frame_pointers_raw, offset, size - start_pointers_index);
 }
 
 std::string StackTrace::toString(void** frame_pointers_raw, size_t offset, size_t size) {

be/src/common/stack_trace.h

@@ -73,7 +73,7 @@ class StackTrace {
     [[nodiscard]] constexpr size_t getSize() const { return size; }
     [[nodiscard]] constexpr size_t getOffset() const { return offset; }
     [[nodiscard]] const FramePointers& getFramePointers() const { return frame_pointers; }
-    [[nodiscard]] std::string toString() const;
+    [[nodiscard]] std::string toString(int start_pointers_index = 0) const;
 
     static std::string toString(void** frame_pointers, size_t offset, size_t size);
     static void dropCache();

be/src/common/status.h

@@ -381,7 +381,7 @@ class Status {
         }
 #ifdef ENABLE_STACKTRACE
         if (stacktrace && capture_stacktrace(code)) {
-            status._err_msg->_stack = get_stack_trace();
+            status._err_msg->_stack = get_stack_trace(1);
             LOG(WARNING) << "meet error status: " << status; // may print too many stacks.
         }
 #endif

be/src/util/stack_util.cpp

@@ -35,7 +35,7 @@ void DumpStackTraceToString(std::string* stacktrace);
 
 namespace doris {
 
-std::string get_stack_trace() {
+std::string get_stack_trace(int start_pointers_index) {
 #ifdef ENABLE_STACKTRACE
     auto tool = config::get_stack_trace_tool;
     if (tool == "glog") {
@@ -48,7 +48,7 @@ std::string get_stack_trace() {
 #if defined(__APPLE__) // TODO
         return get_stack_trace_by_glog();
 #endif
-        return get_stack_trace_by_libunwind();
+        return get_stack_trace_by_libunwind(start_pointers_index);
     } else {
         return "no stack";
     }
@@ -80,8 +80,8 @@ std::string get_stack_trace_by_glibc() {
     return out.str();
 }
 
-std::string get_stack_trace_by_libunwind() {
-    return "\n" + StackTrace().toString();
+std::string get_stack_trace_by_libunwind(int start_pointers_index) {
+    return "\n" + StackTrace().toString(start_pointers_index);
 }
 
 } // namespace doris

be/src/util/stack_util.h

@@ -29,7 +29,7 @@ namespace doris {
 // boost: 1000 times cost 1min, has line numbers, but has memory leak.
 // glibc: 1000 times cost 1min, no line numbers, unresolved backtrace symbol.
 // libunwind: cost is negligible, has line numbers.
-std::string get_stack_trace();
+std::string get_stack_trace(int start_pointers_index = 0);
 
 // Note: there is a libc bug that causes this not to work on 64 bit machines
 // for recursive calls.
@@ -53,6 +53,6 @@ std::string get_stack_trace_by_glibc();
 //  2. Support signal handle
 //  3. libunwid support unw_backtrace for jemalloc
 //  4. Use of undefined compile option USE_MUSL for later
-std::string get_stack_trace_by_libunwind();
+std::string get_stack_trace_by_libunwind(int start_pointers_index);
 
 } // namespace doris