From cc854a32f1df30161939b5456af5046fbfd81a91 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Thu, 30 May 2024 18:29:34 +0800 Subject: [PATCH] [fix](libjdk) Revert support loading libjvm at runtime (#35557) ## Proposed changes Revert #13660 `-l` and `dlopen` load Dynamic Link Library, will use the Allocator of main program, Doris use Jemalloc by default. but, `dlopen` load libjvm.so and use Jemalloc compiled with prefix, overwriting malloc/free will incompatible with libjvm.so. see #34578 for details. In addition, jemalloc not recommend `dlopen` to load Dynamic Link Library, `dlclose` will memory leak in that case. https://github.com/jemalloc/jemalloc/issues/2404 https://github.com/jemalloc/jemalloc/issues/1321 https://github.com/jemalloc/jemalloc/issues/1890 `export LD_PRELOAD` to force `libjvm.so` use a separate `jemalloc.so` will not solve the problem, but may cause a new crash. ``` received by PID 2368 (TID 2391 OR 0x7f445cafb700) from PID 0; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /root/selectdb-core/be/src/common/signal_handler.h:421 1# PosixSignals::chained_handler(int, siginfo*, void*) [clone .part.0] in /opt/jdk/lib/server/libjvm.so 2# JVM_handle_linux_signal in /opt/jdk/lib/server/libjvm.so 3# 0x00007F448DB40400 in /lib64/libc.so.6 4# je_arena_dalloc_promoted at /root/selectdb-core/thirdparty/src/jemalloc-5.3.0/doris_build/../src/arena.c:1277 5# je_free_default at /root/selectdb-core/thirdparty/src/jemalloc-5.3.0/doris_build/../src/jemalloc.c:3014 6# __pthread_create_2_1 in /lib64/libpthread.so.0 7# os::create_thread(Thread*, os::ThreadType, unsigned long) in /opt/jdk/lib/server/libjvm.so 8# CompilerThread::CompilerThread(CompileQueue*, CompilerCounters*) in /opt/jdk/lib/server/libjvm.so 9# CompileBroker::make_thread(CompileBroker::ThreadType, _jobject*, CompileQueue*, AbstractCompiler*, JavaThread*) [clone .constprop.0] in /opt/jdk/lib/server/libjvm.so 10# CompileBroker::possibly_add_compiler_threads(JavaThread*) in /opt/jdk/lib/server/libjvm.so 11# CompileBroker::compiler_thread_loop() in /opt/jdk/lib/server/libjvm.so 12# JavaThread::thread_main_inner() in /opt/jdk/lib/server/libjvm.so 13# Thread::call_run() in /opt/jdk/lib/server/libjvm.so 14# thread_native_entry(Thread*) in /opt/jdk/lib/server/libjvm.so 15# start_thread in /lib64/libpthread.so.0 16# clone in /lib64/libc.so.6 ``` --- be/CMakeLists.txt | 22 +++++-- be/src/util/CMakeLists.txt | 2 + be/src/util/jni-util.cpp | 4 +- bin/start_be.sh | 115 +++++++++++++++++++++++++------------ run-be-ut.sh | 29 ++++++++++ tools/find_libjvm.sh | 61 ++++++++++++++++++++ 6 files changed, 190 insertions(+), 43 deletions(-) create mode 100755 tools/find_libjvm.sh diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 9dd71d40245fa8..fafaff15fe164c 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -443,11 +443,25 @@ if ("${DORIS_JAVA_HOME}" STREQUAL "") set(DORIS_JAVA_HOME "$ENV{JAVA_HOME}") endif() -include_directories(${DORIS_JAVA_HOME}/include) -if (NOT OS_MACOSX) - include_directories(${DORIS_JAVA_HOME}/include/linux) +execute_process(COMMAND chmod 755 ${BASE_DIR}/../tools/find_libjvm.sh) +execute_process(COMMAND ${BASE_DIR}/../tools/find_libjvm.sh OUTPUT_VARIABLE LIBJVM_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) +FILE(GLOB_RECURSE LIB_JVM ${LIBJVM_PATH}) +if("${LIB_JVM}" STREQUAL "") + message(STATUS "there is no libjvm found!") else() - include_directories(${DORIS_JAVA_HOME}/include/darwin) + set(DORIS_DEPENDENCIES + ${DORIS_DEPENDENCIES} + jvm + ) + add_library(jvm SHARED IMPORTED) + set_target_properties(jvm PROPERTIES IMPORTED_LOCATION ${LIB_JVM}) + include_directories(${DORIS_JAVA_HOME}/include) + if (NOT OS_MACOSX) + include_directories(${DORIS_JAVA_HOME}/include/linux) + else() + include_directories(${DORIS_JAVA_HOME}/include/darwin) + endif() + add_definitions("-DLIBJVM") endif() if (NOT OS_MACOSX) diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 8b99fcf015eb36..de4d27eeebb708 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -29,6 +29,8 @@ else () list(REMOVE_ITEM UTIL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/perf_counters_mac.cpp ${CMAKE_CURRENT_SOURCE_DIR}/disk_info_mac.cpp) endif() +list(REMOVE_ITEM UTIL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/libjvm_loader.cpp) + add_library(Util STATIC ${UTIL_FILES} ) diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index 428c587f667711..25f4f913d8a4cb 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -37,7 +37,7 @@ #include "gutil/strings/substitute.h" #include "util/doris_metrics.h" #include "util/jni_native_method.h" -#include "util/libjvm_loader.h" +// #include "util/libjvm_loader.h" using std::string; @@ -456,7 +456,7 @@ Status JniUtil::get_jni_scanner_class(JNIEnv* env, const char* classname, } Status JniUtil::Init() { - RETURN_IF_ERROR(LibJVMLoader::instance().load()); + // RETURN_IF_ERROR(LibJVMLoader::instance().load()); // Get the JNIEnv* corresponding to current thread. JNIEnv* env = nullptr; diff --git a/bin/start_be.sh b/bin/start_be.sh index 361f37552e871d..7d3b5c5679ebac 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -31,12 +31,14 @@ OPTS="$(getopt \ -o '' \ -l 'daemon' \ -l 'console' \ + -l 'version' \ -- "$@")" eval set -- "${OPTS}" RUN_DAEMON=0 RUN_CONSOLE=0 +RUN_VERSION=0 while true; do case "$1" in --daemon) @@ -47,6 +49,10 @@ while true; do RUN_CONSOLE=1 shift ;; + --version) + RUN_VERSION=1 + shift + ;; --) shift break @@ -64,6 +70,78 @@ DORIS_HOME="$( )" export DORIS_HOME +# export env variables from be.conf +# +# LOG_DIR +# PID_DIR +export LOG_DIR="${DORIS_HOME}/log" +PID_DIR="$( + cd "${curdir}" + pwd +)" +export PID_DIR + +jdk_version() { + local java_cmd="${1}" + local result + local IFS=$'\n' + + if ! command -v "${java_cmd}" >/dev/null; then + echo "ERROR: invalid java_cmd ${java_cmd}" >>"${LOG_DIR}/be.out" + result=no_java + return 1 + else + echo "INFO: java_cmd ${java_cmd}" >>"${LOG_DIR}/be.out" + local version + # remove \r for Cygwin + version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" + version="${version//\"/}" + if [[ "${version}" =~ ^1\. ]]; then + result="$(echo "${version}" | awk -F '.' '{print $2}')" + else + result="$(echo "${version}" | awk -F '.' '{print $1}')" + fi + echo "INFO: jdk_version ${result}" >>"${LOG_DIR}/be.out" + fi + echo "${result}" + return 0 +} + +setup_java_env() { + local java_version + + if [[ -z "${JAVA_HOME}" ]]; then + return 1 + fi + + local jvm_arch='amd64' + if [[ "$(uname -m)" == 'aarch64' ]]; then + jvm_arch='aarch64' + fi + java_version="$( + set -e + jdk_version "${JAVA_HOME}/bin/java" + )" + if [[ "${java_version}" -gt 8 ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${JAVA_HOME}/lib:${LD_LIBRARY_PATH}" + # JAVA_HOME is jdk + elif [[ -d "${JAVA_HOME}/jre" ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}/server:${JAVA_HOME}/jre/lib/${jvm_arch}:${LD_LIBRARY_PATH}" + # JAVA_HOME is jre + else + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/${jvm_arch}/server:${JAVA_HOME}/lib/${jvm_arch}:${LD_LIBRARY_PATH}" + fi +} + +# prepare jvm if needed +setup_java_env || true + +if [[ "${RUN_VERSION}" -eq 1 ]]; then + chmod 755 "${DORIS_HOME}/lib/doris_be" + "${DORIS_HOME}"/lib/doris_be --version + exit 1 +fi + if [[ "$(uname -s)" != 'Darwin' ]]; then MAX_MAP_COUNT="$(cat /proc/sys/vm/max_map_count)" if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then @@ -144,43 +222,6 @@ export DORIS_CLASSPATH="-Djava.class.path=${DORIS_CLASSPATH}" export LD_LIBRARY_PATH="${DORIS_HOME}/lib/hadoop_hdfs/native:${LD_LIBRARY_PATH}" -jdk_version() { - local java_cmd="${1}" - local result - local IFS=$'\n' - - if ! command -v "${java_cmd}" >/dev/null; then - echo "ERROR: invalid java_cmd ${java_cmd}" >>"${LOG_DIR}/be.out" - result=no_java - return 1 - else - echo "INFO: java_cmd ${java_cmd}" >>"${LOG_DIR}/be.out" - local version - # remove \r for Cygwin - version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" - version="${version//\"/}" - if [[ "${version}" =~ ^1\. ]]; then - result="$(echo "${version}" | awk -F '.' '{print $2}')" - else - result="$(echo "${version}" | awk -F '.' '{print $1}')" - fi - echo "INFO: jdk_version ${result}" >>"${LOG_DIR}/be.out" - fi - echo "${result}" - return 0 -} - -# export env variables from be.conf -# -# LOG_DIR -# PID_DIR -export LOG_DIR="${DORIS_HOME}/log" -PID_DIR="$( - cd "${curdir}" - pwd -)" -export PID_DIR - # set odbc conf path export ODBCSYSINI="${DORIS_HOME}/conf" diff --git a/run-be-ut.sh b/run-be-ut.sh index 453cd5d397a3f7..74c950ef7424ea 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -278,6 +278,35 @@ while read -r gcda_file; do rm "${gcda_file}" done < <(find "${CMAKE_BUILD_DIR}" -name "*gcda") +setup_java_env() { + local java_version + + if [[ -z "${JAVA_HOME}" ]]; then + return 1 + fi + + local jvm_arch='amd64' + if [[ "$(uname -m)" == 'aarch64' ]]; then + jvm_arch='aarch64' + fi + java_version="$( + set -e + jdk_version "${JAVA_HOME}/bin/java" + )" + if [[ "${java_version}" -gt 8 ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${JAVA_HOME}/lib:${LD_LIBRARY_PATH}" + # JAVA_HOME is jdk + elif [[ -d "${JAVA_HOME}/jre" ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}/server:${JAVA_HOME}/jre/lib/${jvm_arch}:${LD_LIBRARY_PATH}" + # JAVA_HOME is jre + else + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/${jvm_arch}/server:${JAVA_HOME}/lib/${jvm_arch}:${LD_LIBRARY_PATH}" + fi +} + +# prepare jvm if needed +setup_java_env || true + # prepare gtest output dir GTEST_OUTPUT_DIR="${CMAKE_BUILD_DIR}/gtest_output" rm -rf "${GTEST_OUTPUT_DIR}" diff --git a/tools/find_libjvm.sh b/tools/find_libjvm.sh new file mode 100755 index 00000000000000..79235a2b440326 --- /dev/null +++ b/tools/find_libjvm.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +jdk_version() { + local java_cmd="${1}" + local result + local IFS=$'\n' + + if [[ -z "${java_cmd}" ]]; then + result=no_java + return 1 + else + local version + # remove \r for Cygwin + version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" + version="${version//\"/}" + if [[ "${version}" =~ ^1\. ]]; then + result="$(echo "${version}" | awk -F '.' '{print $2}')" + else + result="$(echo "${version}" | awk -F '.' '{print $1}')" + fi + fi + echo "${result}" + return 0 +} + +java_version=$(jdk_version "${JAVA_HOME:-}/bin/java") +jvm_arch='amd64' +if [[ "$(uname -m)" == 'aarch64' ]]; then + jvm_arch='aarch64' +fi +if [[ "${java_version}" -gt 8 ]]; then + export LIBJVM_PATH="${JAVA_HOME}/lib" +# JAVA_HOME is jdk +elif [[ -d "${JAVA_HOME}/jre" ]]; then + export LIBJVM_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}" +# JAVA_HOME is jre +else + export LIBJVM_PATH="${JAVA_HOME}/lib/${jvm_arch}" +fi + +if [[ "$(uname -s)" != 'Darwin' ]]; then + echo "${LIBJVM_PATH}"/*/libjvm.so +else + echo "${LIBJVM_PATH}"/*/libjvm.dylib +fi