Revert Fix jdk17 and jemalloc hook not compatible on some envs

xinyiZzz · May 31, 2024 · 607c49b · 607c49b
1 parent 8651751
commit 607c49b
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 3 deletions.
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
@@ -20,7 +20,10 @@
 // IWYU pragma: no_include <bthread/errno.h>
 #include <errno.h> // IWYU pragma: keep
 #include <gflags/gflags.h>
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/malloc_extension.h> // IWYU pragma: keep
+#endif
 // IWYU pragma: no_include <bits/std_abs.h>
 #include <butil/iobuf.h>
 #include <math.h>

diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp
@@ -17,9 +17,12 @@
 
 #include "http/action/pprof_actions.h"
 
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/heap-profiler.h>    // IWYU pragma: keep
 #include <gperftools/malloc_extension.h> // IWYU pragma: keep
 #include <gperftools/profiler.h>         // IWYU pragma: keep
+#endif
 #include <stdio.h>
 
 #include <fstream>

diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
@@ -22,7 +22,9 @@
 #include <boost/algorithm/string/replace.hpp>
 #ifdef USE_JEMALLOC
 #include "jemalloc/jemalloc.h"
-#else
+#endif
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/malloc_extension.h>
 #endif
 

diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h
@@ -231,11 +231,14 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/port.h>
 
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 // gperftools headers
 #include <gperftools/malloc_extension.h>
 #include <gperftools/malloc_hook.h>
 #include <gperftools/nallocx.h>
 #include <gperftools/tcmalloc.h>
+#endif
 
 // hs headers
 #include <hs/hs.h>

diff --git a/be/src/runtime/memory/tcmalloc_hook.h b/be/src/runtime/memory/tcmalloc_hook.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/malloc_hook.h>
 #include <gperftools/nallocx.h>
 #include <gperftools/tcmalloc.h>
@@ -53,3 +55,4 @@ void init_hook() {
 //     MallocHook::RemoveNewHook(&new_hook);
 //     MallocHook::RemoveDeleteHook(&delete_hook);
 // }
+#endif
diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h
@@ -93,6 +93,7 @@
 
 // Count a code segment memory (memory malloc - memory free) to MemTracker.
 // Compared to count `scope_mem`, MemTracker is easier to observe from the outside and is thread-safe.
+// Usage example: std::unique_ptr<MemTracker> tracker = std::make_unique<MemTracker>("first_tracker");
 //                { SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(_mem_tracker.get()); xxx; xxx; }
 #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \
     auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker)
@@ -107,6 +108,8 @@
     auto VARNAME_LINENUM(scope_skip_memory_check) = doris::ScopeSkipMemoryCheck()
 
 #define SKIP_LARGE_MEMORY_CHECK(...)                                       \
+    do {                                                                   \
+        doris::ThreadLocalHandle::create_thread_local_if_not_exits();      \
         doris::thread_context()->skip_large_memory_check++;                \
         DEFER({                                                            \
             doris::thread_context()->skip_large_memory_check--;            \

diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
@@ -24,7 +24,10 @@
 // IWYU pragma: no_include <bthread/errno.h>
 #include <errno.h> // IWYU pragma: keep
 #include <fcntl.h>
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/malloc_extension.h> // IWYU pragma: keep
+#endif
 #include <libgen.h>
 #include <setjmp.h>
 #include <signal.h>

diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
@@ -36,7 +36,9 @@
 #include "common/logging.h"
 #ifdef USE_JEMALLOC
 #include "jemalloc/jemalloc.h"
-#else
+#endif
+#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \
+        !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC)
 #include <gperftools/malloc_extension.h>
 #endif
 #include "common/config.h"

diff --git a/bin/start_be.sh b/bin/start_be.sh
@@ -387,9 +387,11 @@ fi
 
 if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then
     export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:"
+    export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:"
 else
     JEMALLOC_PROF_PRFIX="${DORIS_HOME}/log/${JEMALLOC_PROF_PRFIX}"
     export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}"
+    export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}"
 fi
 
 if [[ "${RUN_DAEMON}" -eq 1 ]]; then

diff --git a/build.sh b/build.sh
@@ -30,6 +30,11 @@ set -eo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 
 export DORIS_HOME="${ROOT}"
+if [[ -z "${DORIS_THIRDPARTY}" ]]; then
+    export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
+fi
+export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
+export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
 
 . "${DORIS_HOME}/env.sh"
 

diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake
@@ -61,6 +61,7 @@ add_thirdparty(thrift)
 add_thirdparty(crypto)
 add_thirdparty(openssl LIBNAME "lib/libssl.a")
 add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a")
+add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a")
 add_thirdparty(leveldb) # Required by brpc
 add_thirdparty(brpc LIB64)
 add_thirdparty(rocksdb) # For local storage mocking

diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh
@@ -128,7 +128,7 @@ sudo docker run -i --rm \
     -v "${teamcity_build_checkoutDir}":/root/doris \
     "${docker_image}" \
     /bin/bash -c "mkdir -p ${git_storage_path} \
-    				&& cp -r /root/git/* ${git_storage_path}/ \
+                    && cp -r /root/git/* ${git_storage_path}/ \
                     && cd /root/doris \
                     && export CCACHE_LOGFILE=/tmp/cache.debug \
                     && export CCACHE_REMOTE_STORAGE=file:///root/ccache \