diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 059910be3ef9bed..647f0275e7a39bc 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -20,7 +20,10 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep +#endif // IWYU pragma: no_include #include #include diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index 1cbe2163b9374f7..2894ba017813827 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -17,9 +17,12 @@ #include "http/action/pprof_actions.h" +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep #include // IWYU pragma: keep #include // IWYU pragma: keep +#endif #include #include diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index fa46ca9d04ead25..5d0689e2a8a7a94 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -22,7 +22,9 @@ #include #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#else +#endif +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #endif diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h index 8c7ef3ea3a9fb87..ca6fbea84634829 100644 --- a/be/src/pch/pch.h +++ b/be/src/pch/pch.h @@ -231,11 +231,14 @@ #include #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) // gperftools headers #include #include #include #include +#endif // hs headers #include diff --git a/be/src/runtime/memory/tcmalloc_hook.h b/be/src/runtime/memory/tcmalloc_hook.h index 9cc88be1b081bc0..9f7bee16b0fe78d 100644 --- a/be/src/runtime/memory/tcmalloc_hook.h +++ b/be/src/runtime/memory/tcmalloc_hook.h @@ -17,6 +17,8 @@ #pragma once +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #include #include @@ -53,3 +55,4 @@ void init_hook() { // MallocHook::RemoveNewHook(&new_hook); // MallocHook::RemoveDeleteHook(&delete_hook); // } +#endif diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index 20346be62384d37..45f64a3739ae1ac 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -93,6 +93,7 @@ // Count a code segment memory (memory malloc - memory free) to MemTracker. // Compared to count `scope_mem`, MemTracker is easier to observe from the outside and is thread-safe. +// Usage example: std::unique_ptr tracker = std::make_unique("first_tracker"); // { SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(_mem_tracker.get()); xxx; xxx; } #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker) @@ -107,6 +108,8 @@ auto VARNAME_LINENUM(scope_skip_memory_check) = doris::ScopeSkipMemoryCheck() #define SKIP_LARGE_MEMORY_CHECK(...) \ + do { \ + doris::ThreadLocalHandle::create_thread_local_if_not_exits(); \ doris::thread_context()->skip_large_memory_check++; \ DEFER({ \ doris::thread_context()->skip_large_memory_check--; \ diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index ebc515cc12810ce..92d3452dcb136d9 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -24,7 +24,10 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep +#endif #include #include #include diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index dc4b0e0d2986a42..82de2856853c32c 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -36,7 +36,9 @@ #include "common/logging.h" #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#else +#endif +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #endif #include "common/config.h" diff --git a/bin/start_be.sh b/bin/start_be.sh index 4f4eb2b1bccc853..7d3b5c5679ebac4 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -387,9 +387,11 @@ fi if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" + export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" else JEMALLOC_PROF_PRFIX="${DORIS_HOME}/log/${JEMALLOC_PROF_PRFIX}" export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" + export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" fi if [[ "${RUN_DAEMON}" -eq 1 ]]; then diff --git a/build.sh b/build.sh index 690d617d510ec89..321c27ee1acfa81 100755 --- a/build.sh +++ b/build.sh @@ -30,6 +30,11 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export DORIS_HOME="${ROOT}" +if [[ -z "${DORIS_THIRDPARTY}" ]]; then + export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty" +fi +export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include" +export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib" . "${DORIS_HOME}/env.sh" diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index 83c10d2be9ed427..0e148896bfcc1e6 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -61,6 +61,7 @@ add_thirdparty(thrift) add_thirdparty(crypto) add_thirdparty(openssl LIBNAME "lib/libssl.a") add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") +add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a") add_thirdparty(leveldb) # Required by brpc add_thirdparty(brpc LIB64) add_thirdparty(rocksdb) # For local storage mocking diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh index 76ab06574cbed32..3a668cd3fdfdd3a 100644 --- a/regression-test/pipeline/performance/compile.sh +++ b/regression-test/pipeline/performance/compile.sh @@ -128,7 +128,7 @@ sudo docker run -i --rm \ -v "${teamcity_build_checkoutDir}":/root/doris \ "${docker_image}" \ /bin/bash -c "mkdir -p ${git_storage_path} \ - && cp -r /root/git/* ${git_storage_path}/ \ + && cp -r /root/git/* ${git_storage_path}/ \ && cd /root/doris \ && export CCACHE_LOGFILE=/tmp/cache.debug \ && export CCACHE_REMOTE_STORAGE=file:///root/ccache \