diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 7016fc8a9dd817..9dd71d40245fa8 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -72,6 +72,7 @@ option(USE_LIBCPP "Use libc++" OFF) option(USE_MEM_TRACKER, "Use memory tracker" ON) option(USE_UNWIND "Use libunwind" ON) option(USE_JEMALLOC "Use jemalloc" ON) +option(USE_JEMALLOC_HOOK "Use jemalloc hook" ON) if (OS_MACOSX) set(GLIBC_COMPATIBILITY OFF) set(USE_LIBCPP ON) @@ -87,6 +88,7 @@ message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}") message(STATUS "USE_LIBCPP is ${USE_LIBCPP}") message(STATUS "USE_MEM_TRACKER is ${USE_MEM_TRACKER}") message(STATUS "USE_JEMALLOC is ${USE_JEMALLOC}") +message(STATUS "USE_JEMALLOC_HOOK is ${USE_JEMALLOC_HOOK}") message(STATUS "USE_UNWIND is ${USE_UNWIND}") message(STATUS "ENABLE_PCH is ${ENABLE_PCH}") @@ -345,6 +347,9 @@ endif() if (USE_JEMALLOC) add_definitions(-DUSE_JEMALLOC) endif() +if (USE_JEMALLOC_HOOK) + add_definitions(-DUSE_JEMALLOC_HOOK) +endif() # Compile with libunwind if (USE_UNWIND) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 3a217413db8feb..94c45dd526f979 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -20,7 +20,10 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep +#endif // IWYU pragma: no_include #include #include diff --git a/be/src/http/action/jeprofile_actions.cpp b/be/src/http/action/jeprofile_actions.cpp index 1ed763cfb747f4..703902096bc9ab 100644 --- a/be/src/http/action/jeprofile_actions.cpp +++ b/be/src/http/action/jeprofile_actions.cpp @@ -60,7 +60,11 @@ void JeHeapAction::handle(HttpRequest* req) { << "." << rand() << ".heap"; const std::string& tmp_file_name_str = tmp_jeprof_file_name.str(); const char* file_name_ptr = tmp_file_name_str.c_str(); +#ifdef USE_JEMALLOC_HOOK int result = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*)); +#else + int result = mallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*)); +#endif std::stringstream response; if (result == 0) { response << "Jemalloc heap dump success, dump file path: " << tmp_jeprof_file_name.str() diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index 1cbe2163b9374f..2894ba01781382 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -17,9 +17,12 @@ #include "http/action/pprof_actions.h" +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep #include // IWYU pragma: keep #include // IWYU pragma: keep +#endif #include #include diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index fa46ca9d04ead2..f9003373f8257e 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -22,7 +22,9 @@ #include #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#else +#endif +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #endif @@ -112,7 +114,11 @@ void mem_usage_handler(const WebPageHandler::ArgumentMap& args, std::stringstrea auto* _opaque = static_cast(opaque); _opaque->append(buf); }; +#ifdef USE_JEMALLOC_HOOK jemalloc_stats_print(write_cb, &tmp, "a"); +#else + malloc_stats_print(write_cb, &tmp, "a"); +#endif boost::replace_all(tmp, "\n", "
"); (*output) << tmp << ""; #else diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h index 8c7ef3ea3a9fb8..ca6fbea8463482 100644 --- a/be/src/pch/pch.h +++ b/be/src/pch/pch.h @@ -231,11 +231,14 @@ #include #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) // gperftools headers #include #include #include #include +#endif // hs headers #include diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index a0b3b799a764cb..70db5b4cb52cfd 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -25,7 +25,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime") file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc) -if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER) +if (NOT USE_JEMALLOC OR NOT USE_JEMALLOC_HOOK) list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp) endif() diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 7a3170b73707df..77648c6cab4ea3 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -837,6 +837,26 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params, std::shared_ptr query_ctx; RETURN_IF_ERROR(_get_query_ctx(params, params.query_id, true, query_ctx)); SCOPED_ATTACH_TASK_WITH_ID(query_ctx->query_mem_tracker, params.query_id); + LOG(INFO) << "222222 "; +#ifdef USE_JEMALLOC_HOOK + int64_t scope_mem = 0; + void* ptr1; + { + SCOPED_MEM_COUNT_BY_HOOK(&scope_mem); + ptr1 = malloc(1000); + } + free(ptr1); + LOG(INFO) << "111111 " << std::to_string(scope_mem); +#else + int64_t scope_mem = 0; + void* ptr1; + { + SCOPED_MEM_COUNT_BY_HOOK(&scope_mem); + ptr1 = malloc(1000); + } + free(ptr1); + LOG(INFO) << "111111 " << std::to_string(scope_mem); +#endif DCHECK((params.query_options.__isset.enable_pipeline_x_engine && params.query_options.enable_pipeline_x_engine) || (params.query_options.__isset.enable_pipeline_engine && diff --git a/be/src/runtime/memory/tcmalloc_hook.h b/be/src/runtime/memory/tcmalloc_hook.h index 9cc88be1b081bc..9f7bee16b0fe78 100644 --- a/be/src/runtime/memory/tcmalloc_hook.h +++ b/be/src/runtime/memory/tcmalloc_hook.h @@ -17,6 +17,8 @@ #pragma once +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #include #include @@ -53,3 +55,4 @@ void init_hook() { // MallocHook::RemoveNewHook(&new_hook); // MallocHook::RemoveDeleteHook(&delete_hook); // } +#endif diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index bc221ee0dcf62c..48a7912e0d70f9 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -59,18 +59,6 @@ #define SCOPED_CONSUME_MEM_TRACKER(mem_tracker) \ auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumer(mem_tracker) -// Count a code segment memory (memory malloc - memory free) to int64_t -// Usage example: int64_t scope_mem = 0; { SCOPED_MEM_COUNT(&scope_mem); xxx; xxx; } -#define SCOPED_MEM_COUNT_BY_HOOK(scope_mem) \ - auto VARNAME_LINENUM(scope_mem_count) = doris::ScopeMemCountByHook(scope_mem) - -// Count a code segment memory (memory malloc - memory free) to MemTracker. -// Compared to count `scope_mem`, MemTracker is easier to observe from the outside and is thread-safe. -// Usage example: std::unique_ptr tracker = std::make_unique("first_tracker"); -// { SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(_mem_tracker.get()); xxx; xxx; } -#define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ - auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker) - #define ORPHAN_TRACKER_CHECK() \ DCHECK(doris::k_doris_exit || !doris::config::enable_memory_orphan_check || \ doris::thread_context()->thread_mem_tracker()->label() != "Orphan") \ @@ -89,12 +77,27 @@ auto VARNAME_LINENUM(scoped_tls_stmtl) = doris::ScopedInitThreadContext() #define SCOPED_CONSUME_MEM_TRACKER(mem_tracker) \ auto VARNAME_LINENUM(scoped_tls_cmt) = doris::ScopedInitThreadContext() +#define ORPHAN_TRACKER_CHECK() (void)0 +#define MEMORY_ORPHAN_CHECK() (void)0 +#endif + +#if defined(USE_MEM_TRACKER) && !defined(BE_TEST) && defined(USE_JEMALLOC_HOOK) +// Count a code segment memory (memory malloc - memory free) to int64_t +// Usage example: int64_t scope_mem = 0; { SCOPED_MEM_COUNT_BY_HOOK(&scope_mem); xxx; xxx; } +#define SCOPED_MEM_COUNT_BY_HOOK(scope_mem) \ + auto VARNAME_LINENUM(scope_mem_count) = doris::ScopeMemCountByHook(scope_mem) + +// Count a code segment memory (memory malloc - memory free) to MemTracker. +// Compared to count `scope_mem`, MemTracker is easier to observe from the outside and is thread-safe. +// Usage example: std::unique_ptr tracker = std::make_unique("first_tracker"); +// { SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(_mem_tracker.get()); xxx; xxx; } +#define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ + auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker) +#else #define SCOPED_MEM_COUNT_BY_HOOK(scope_mem) \ auto VARNAME_LINENUM(scoped_tls_mcbh) = doris::ScopedInitThreadContext() #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ auto VARNAME_LINENUM(scoped_tls_cmtbh) = doris::ScopedInitThreadContext() -#define ORPHAN_TRACKER_CHECK() (void)0 -#define MEMORY_ORPHAN_CHECK() (void)0 #endif #define SCOPED_SKIP_MEMORY_CHECK() \ diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index ebc515cc12810c..92d3452dcb136d 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -24,7 +24,10 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep +#endif #include #include #include diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index ce42d69dccf7b6..43c2c1212a4cd7 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -89,7 +89,11 @@ void MemInfo::refresh_allocator_mem() { // the current epoch number, which might be useful to log as a sanity check. uint64_t epoch = 0; size_t sz = sizeof(epoch); +#ifdef USE_JEMALLOC_HOOK jemallctl("epoch", &epoch, &sz, &epoch, sz); +#else + mallctl("epoch", &epoch, &sz, &epoch, sz); +#endif // https://jemalloc.net/jemalloc.3.html // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index f6c9407d73eb58..0abfb2b34f3cf0 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -36,7 +36,9 @@ #include "common/logging.h" #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#else +#endif +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #endif #include "common/config.h" @@ -105,9 +107,15 @@ class MemInfo { #ifdef USE_JEMALLOC size_t value = 0; size_t sz = sizeof(value); +#ifdef USE_JEMALLOC_HOOK if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) { return value; } +#else + if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) { + return value; + } +#endif #endif return 0; } @@ -128,8 +136,13 @@ class MemInfo { if (config::enable_je_purge_dirty_pages) { try { // Purge all unused dirty pages for arena , or for all arenas if equals MALLCTL_ARENAS_ALL. +#ifdef USE_JEMALLOC_HOOK jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr, nullptr, 0); +#else + mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr, + nullptr, 0); +#endif } catch (...) { LOG(WARNING) << "Purge all unused dirty pages for all arenas failed"; } diff --git a/bin/start_be.sh b/bin/start_be.sh index 743c2f98a97ec6..361f37552e871d 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -346,9 +346,11 @@ fi if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" + export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" else JEMALLOC_PROF_PRFIX="${DORIS_HOME}/log/${JEMALLOC_PROF_PRFIX}" export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" + export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" fi if [[ "${RUN_DAEMON}" -eq 1 ]]; then diff --git a/build.sh b/build.sh index e1822164b78d66..905bb54406246b 100755 --- a/build.sh +++ b/build.sh @@ -30,6 +30,10 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export DORIS_HOME="${ROOT}" +export TP_DIR="${ROOT}/thirdparty" +export TP_INSTALL_DIR="${TP_DIR:-.}/installed" +export TP_INCLUDE_DIR="${TP_INSTALL_DIR}/include" +export TP_LIB_DIR="${TP_INSTALL_DIR}/lib" . "${DORIS_HOME}/env.sh" @@ -357,6 +361,28 @@ fi if [[ -z "${USE_JEMALLOC}" ]]; then USE_JEMALLOC='ON' fi +if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then + USE_JEMALLOC_HOOK='OFF' +fi +# update jemalloc prefix +rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" +rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" +rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" +rm -rf "${TP_INCLUDE_DIR}/rocksdb" +rm -rf "${TP_LIB_DIR}/librocksdb.a" +if [[ "${USE_JEMALLOC_HOOK}" == "ON" ]]; then + cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" + cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + cp "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a" + cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb" +else + cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_no_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" + cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + cp "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" "${TP_LIB_DIR}/librocksdb.a" + cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" "${TP_INCLUDE_DIR}/rocksdb" +fi if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then USE_BTHREAD_SCANNER='OFF' fi @@ -462,6 +488,7 @@ echo "Get params: STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO} USE_MEM_TRACKER -- ${USE_MEM_TRACKER} USE_JEMALLOC -- ${USE_JEMALLOC} + USE_JEMALLOC_HOOK -- ${USE_JEMALLOC_HOOK} USE_BTHREAD_SCANNER -- ${USE_BTHREAD_SCANNER} ENABLE_STACKTRACE -- ${ENABLE_STACKTRACE} ENABLE_INJECTION_POINT -- ${ENABLE_INJECTION_POINT} @@ -564,6 +591,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_PCH="${ENABLE_PCH}" \ -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ + -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \ -DENABLE_STACKTRACE="${ENABLE_STACKTRACE}" \ -DUSE_AVX2="${USE_AVX2}" \ -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ @@ -607,6 +635,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \ -DUSE_DWARF="${USE_DWARF}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ + -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \ "${DORIS_HOME}/cloud/" diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index 35164af9f9c546..9b9929ae1d5e0c 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -188,6 +188,9 @@ endif () if (USE_JEMALLOC) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC") endif() +if (USE_JEMALLOC_HOOK) + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC_HOOK") +endif() if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new") diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index 83c10d2be9ed42..0e148896bfcc1e 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -61,6 +61,7 @@ add_thirdparty(thrift) add_thirdparty(crypto) add_thirdparty(openssl LIBNAME "lib/libssl.a") add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") +add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a") add_thirdparty(leveldb) # Required by brpc add_thirdparty(brpc LIB64) add_thirdparty(rocksdb) # For local storage mocking diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt index a2abfb075bf934..b18947b04a1e5b 100644 --- a/cloud/src/common/CMakeLists.txt +++ b/cloud/src/common/CMakeLists.txt @@ -16,7 +16,7 @@ set(COMMON_FILES network_util.cpp ) -if (USE_JEMALLOC) +if (USE_JEMALLOC AND USE_JEMALLOC_HOOK) set(COMMON_FILES ${COMMON_FILES} jemalloc_hook.cpp ) diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh index cfeae6af5f2d36..23c2bb3590f019 100644 --- a/regression-test/pipeline/performance/compile.sh +++ b/regression-test/pipeline/performance/compile.sh @@ -129,13 +129,14 @@ sudo docker run -i --rm \ -v "${teamcity_build_checkoutDir}":/root/doris \ "${docker_image}" \ /bin/bash -c "mkdir -p ${git_storage_path} \ - && cp -r /root/git/* ${git_storage_path}/ \ + && cp -r /root/git/* ${git_storage_path}/ \ && cd /root/doris \ && export CCACHE_LOGFILE=/tmp/cache.debug \ && export CCACHE_REMOTE_STORAGE=file:///root/ccache \ && export EXTRA_CXX_FLAGS=-O3 \ && export USE_JEMALLOC='ON' \ - && export ENABLE_PCH=OFF ${jdk17_str}\ + && export USE_JEMALLOC_HOOK_WITH_PREFIX='OFF' \ + && export ENABLE_PCH=OFF ${jdk17_str}\ && export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \ && bash build.sh --fe --be --clean 2>&1 | tee build.log" set +x diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6ce0c98e1a1937..dab894b63f56cd 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -831,7 +831,7 @@ build_brpc() { } # rocksdb -build_rocksdb() { +build_rocksdb_base() { check_if_source_exist "${ROCKSDB_SOURCE}" cd "${TP_SOURCE_DIR}/${ROCKSDB_SOURCE}" @@ -852,9 +852,29 @@ build_rocksdb() { ${warning_defaulted_function_deleted} ${warning_unused_but_set_variable} -Wno-pessimizing-move -Wno-range-loop-construct" \ LDFLAGS="${ldflags}" \ PORTABLE=1 make USE_RTTI=1 -j "${PARALLEL}" static_lib - cp librocksdb.a ../../installed/lib/librocksdb.a - cp -r include/rocksdb ../../installed/include/ - strip_lib librocksdb.a +} + +build_rocksdb() { + build_rocksdb_base + rm -rf "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" + rm -rf "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" + cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" + cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" + strip_lib librocksdb_jemalloc_no_prefix.a + # for compatibility with previous doris version + rm -rf "${TP_LIB_DIR}/librocksdb.a" + rm -rf "${TP_INCLUDE_DIR}/rocksdb" + cp "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" "${TP_LIB_DIR}/librocksdb.a" + cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" "${TP_INCLUDE_DIR}/rocksdb" +} + +build_rocksdb_jemalloc_with_prefix() { + build_rocksdb_base + rm -rf "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" + rm -rf "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" + cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" + cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" + strip_lib librocksdb_jemalloc_with_prefix.a } # cyrus_sasl @@ -1493,7 +1513,7 @@ build_hdfs3() { strip_lib libhdfs3.a } -# jemalloc +# jemalloc no prefix build_jemalloc_doris() { check_if_source_exist "${JEMALLOC_DORIS_SOURCE}" cd "${TP_SOURCE_DIR}/${JEMALLOC_DORIS_SOURCE}" @@ -1514,12 +1534,55 @@ build_jemalloc_doris() { WITH_LG_PAGE='' fi - CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \ + CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris_no_prefix" "${WITH_LG_PAGE}" \ + --enable-prof --disable-libdl --disable-shared + + make -j "${PARALLEL}" + make install + + # for compatibility with previous + rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" + rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_no_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" + cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" +} + +# jemalloc with prefix +build_jemalloc_doris_with_prefix() { + check_if_source_exist "${JEMALLOC_DORIS_SOURCE}" + cd "${TP_SOURCE_DIR}/${JEMALLOC_DORIS_SOURCE}" + + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + + cflags='-O3 -fno-omit-frame-pointer -fPIC -g' + # Build jemalloc --with-lg-page=16 in order to make the wheel work on both 4k and 64k page arm64 systems. + # Jemalloc compiled on a system with page size 4K can only run on a system with the same page size 4K. + # If it is run on a system with page size > 4K, an error `unsupported system page size`. + # Jemalloc compiled on a system with page size 64K can run on a system with page size < 64K, + # but this will waste more memory. Jemalloc does not support dynamic adaptation to the page size of the system. + # The reason is that jemalloc will perform some optimizations based on the page size when compiling. + if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then + WITH_LG_PAGE='--with-lg-page=16' + else + WITH_LG_PAGE='' + fi + + CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris_with_prefix" "${WITH_LG_PAGE}" \ --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared make -j "${PARALLEL}" make install - mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + + # for compatibility with previous doris version + rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" + rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" + cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" } # libunwind @@ -1818,6 +1881,8 @@ if [[ "${#packages[@]}" -eq 0 ]]; then thrift leveldb brpc + jemalloc_doris_with_prefix + rocksdb_jemalloc_with_prefix jemalloc_doris rocksdb krb5 # before cyrus_sasl