From 2bc196407c66daea14c090b45e13ece02964cd01 Mon Sep 17 00:00:00 2001 From: zhaochangle Date: Tue, 24 Sep 2024 02:06:57 +0800 Subject: [PATCH] 1 --- be/CMakeLists.txt | 61 +++++++++++++++---------- be/benchmark/benchmark_main.cpp | 49 ++++++++++++++++++++ be/src/runtime/memory/jemalloc_hook.cpp | 2 +- be/src/service/CMakeLists.txt | 2 +- build.sh | 17 +++++++ 5 files changed, 106 insertions(+), 25 deletions(-) create mode 100644 be/benchmark/benchmark_main.cpp diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 1d79048f96511c..8563abe227cb56 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -130,6 +130,8 @@ message(STATUS "THIRDPARTY_DIR is ${THIRDPARTY_DIR}") option(MAKE_TEST "ON for make unit test or OFF for not" OFF) message(STATUS "make test: ${MAKE_TEST}") +option(BUILD_BENCHMARK "ON for make google benchmark or OFF for not" OFF) +message(STATUS "make benchmark: ${BUILD_BENCHMARK}") option(WITH_MYSQL "Support access MySQL" ON) @@ -568,7 +570,7 @@ if (OS_MACOSX) ) endif() -if (MAKE_TEST) +if (MAKE_TEST OR BUILD_BENCHMARK) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} benchmark @@ -708,6 +710,11 @@ if (MAKE_TEST) endif() endif () +# use this to avoid some runtime tracker. reuse BE_TEST symbol, no need another. +if (BUILD_BENCHMARK) + add_definitions(-DBE_TEST) +endif() + get_directory_property(COMPILER_FLAGS COMPILE_OPTIONS) get_directory_property(COMPILER_DEFINES COMPILE_DEFINITIONS) message(STATUS "Compiler: ${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}") @@ -754,7 +761,7 @@ add_subdirectory(${SRC_DIR}/http) add_subdirectory(${SRC_DIR}/io) add_subdirectory(${SRC_DIR}/olap) add_subdirectory(${SRC_DIR}/runtime) -add_subdirectory(${SRC_DIR}/service) +add_subdirectory(${SRC_DIR}/service) # this include doris_be add_subdirectory(${SRC_DIR}/udf) add_subdirectory(${SRC_DIR}/cloud) @@ -772,36 +779,44 @@ add_subdirectory(${SRC_DIR}/util) add_subdirectory(${SRC_DIR}/vec) add_subdirectory(${SRC_DIR}/pipeline) +# this include doris_be_test if (MAKE_TEST) add_subdirectory(${TEST_DIR}) endif () add_subdirectory(${COMMON_SRC_DIR}/cpp ${BUILD_DIR}/src/common_cpp) -# Install be -install(DIRECTORY DESTINATION ${OUTPUT_DIR}) -install(DIRECTORY DESTINATION ${OUTPUT_DIR}/bin) -install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf) - -install(FILES - ${BASE_DIR}/../bin/start_be.sh - ${BASE_DIR}/../bin/stop_be.sh - ${BASE_DIR}/../tools/jeprof - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE - GROUP_READ GROUP_WRITE GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE - DESTINATION ${OUTPUT_DIR}/bin) - -install(FILES - ${BASE_DIR}/../conf/be.conf - ${BASE_DIR}/../conf/odbcinst.ini - ${BASE_DIR}/../conf/asan_suppr.conf - ${BASE_DIR}/../conf/lsan_suppr.conf - DESTINATION ${OUTPUT_DIR}/conf) +if(NOT BUILD_BENCHMARK) + # Install be + install(DIRECTORY DESTINATION ${OUTPUT_DIR}) + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/bin) + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf) + + install(FILES + ${BASE_DIR}/../bin/start_be.sh + ${BASE_DIR}/../bin/stop_be.sh + ${BASE_DIR}/../tools/jeprof + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_WRITE GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE + DESTINATION ${OUTPUT_DIR}/bin) + + install(FILES + ${BASE_DIR}/../conf/be.conf + ${BASE_DIR}/../conf/odbcinst.ini + ${BASE_DIR}/../conf/asan_suppr.conf + ${BASE_DIR}/../conf/lsan_suppr.conf + DESTINATION ${OUTPUT_DIR}/conf) +endif() get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) foreach(dir ${dirs}) message(STATUS "dir='${dir}'") endforeach() - +if (BUILD_BENCHMARK) + add_executable(benchmark_test ${BASE_DIR}/benchmark/benchmark_main.cpp) + target_link_libraries(benchmark_test ${DORIS_LINK_LIBS}) + message(STATUS "Add benchmark to build") + install(TARGETS benchmark_test DESTINATION ${OUTPUT_DIR}) +endif() \ No newline at end of file diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp new file mode 100644 index 00000000000000..d3364177e86263 --- /dev/null +++ b/be/benchmark/benchmark_main.cpp @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "vec/columns/column_string.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_string.h" + +namespace doris::vectorized { + +static void Example1(benchmark::State& state) { + state.PauseTiming(); + Block block; + DataTypePtr str_type = std::make_shared(); + std::vector vals {100, "content"}; + state.ResumeTiming(); + + for (auto _ : state) { + auto str_col = ColumnString::create(); + for (auto& v : vals) { + str_col->insert_data(v.data(), v.size()); + } + block.insert({std::move(str_col), str_type, "col"}); + benchmark::DoNotOptimize(block); + } +} +BENCHMARK(Example1); + +} // namespace doris::vectorized + +BENCHMARK_MAIN(); diff --git a/be/src/runtime/memory/jemalloc_hook.cpp b/be/src/runtime/memory/jemalloc_hook.cpp index 445d60d382c270..dffc1344b71dbc 100644 --- a/be/src/runtime/memory/jemalloc_hook.cpp +++ b/be/src/runtime/memory/jemalloc_hook.cpp @@ -60,7 +60,7 @@ void* doris_realloc(void* p, size_t size) __THROW { return nullptr; } -#if USE_MEM_TRACKER +#if defined(USE_MEM_TRACKER) && !defined(BE_TEST) int64_t old_size = jemalloc_usable_size(p); CONSUME_THREAD_MEM_TRACKER_BY_HOOK_WITH_FN( [](size_t size, int64_t old_size) { return jenallocx(size, 0) - old_size; }, size, diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 4ce611345840c1..e44045dffce17e 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -28,7 +28,7 @@ add_library(Service STATIC ${SRC_FILES}) pch_reuse(Service) -if (${MAKE_TEST} STREQUAL "OFF") +if (${MAKE_TEST} STREQUAL "OFF" AND ${BUILD_BENCHMARK} STREQUAL "OFF") add_executable(doris_be doris_main.cpp ) diff --git a/build.sh b/build.sh index 35c989d0b0a4ec..8cdfd56f95df4b 100755 --- a/build.sh +++ b/build.sh @@ -49,6 +49,7 @@ Usage: $0 --meta-tool build Backend meta tool. Default OFF. --cloud build Cloud. Default OFF. --index-tool build Backend inverted index tool. Default OFF. + --benchmark build Google Benchmark. Default OFF. --broker build Broker. Default ON. --spark-dpp build Spark DPP application. Default ON. --hive-udf build Hive UDF library for Spark Load. Default ON. @@ -64,12 +65,14 @@ Usage: $0 DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default is OFF. DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE. DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not build azure into BE. + Eg. $0 build all $0 --be build Backend $0 --meta-tool build Backend meta tool $0 --cloud build Cloud $0 --index-tool build Backend inverted index tool + $0 --benchmark build Google Benchmark of Backend $0 --fe --clean clean and build Frontend and Spark Dpp application $0 --fe --be --clean clean and build Frontend, Spark Dpp application and Backend $0 --spark-dpp build Spark DPP application alone @@ -129,6 +132,7 @@ if ! OPTS="$(getopt \ -l 'broker' \ -l 'meta-tool' \ -l 'index-tool' \ + -l 'benchmark' \ -l 'spark-dpp' \ -l 'hive-udf' \ -l 'be-java-extensions' \ @@ -151,6 +155,7 @@ BUILD_CLOUD=0 BUILD_BROKER=0 BUILD_META_TOOL='OFF' BUILD_INDEX_TOOL='OFF' +BUILD_BENCHMARK='OFF' BUILD_SPARK_DPP=0 BUILD_BE_JAVA_EXTENSIONS=0 BUILD_HIVE_UDF=0 @@ -170,6 +175,7 @@ if [[ "$#" == 1 ]]; then BUILD_BROKER=1 BUILD_META_TOOL='OFF' BUILD_INDEX_TOOL='OFF' + BUILD_BENCHMARK='OFF' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 BUILD_BE_JAVA_EXTENSIONS=1 @@ -205,6 +211,11 @@ else BUILD_INDEX_TOOL='ON' shift ;; + --benchmark) + BUILD_BENCHMARK='ON' + BUILD_BE=1 # go into BE cmake building, but benchmark instead of doris_be + shift + ;; --spark-dpp) BUILD_SPARK_DPP=1 shift @@ -446,6 +457,10 @@ if [[ -z "${ENABLE_CACHE_LOCK_DEBUG}" ]]; then ENABLE_CACHE_LOCK_DEBUG='OFF' fi +if [[ -z "${BUILD_BENCHMARK}" ]]; then + BUILD_BENCHMARK='OFF' +fi + if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then RECORD_COMPILER_SWITCHES='OFF' fi @@ -476,6 +491,7 @@ echo "Get params: BUILD_BROKER -- ${BUILD_BROKER} BUILD_META_TOOL -- ${BUILD_META_TOOL} BUILD_INDEX_TOOL -- ${BUILD_INDEX_TOOL} + BUILD_BENCHMARK -- ${BUILD_BENCHMARK} BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP} BUILD_BE_JAVA_EXTENSIONS -- ${BUILD_BE_JAVA_EXTENSIONS} BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF} @@ -581,6 +597,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_INJECTION_POINT="${ENABLE_INJECTION_POINT}" \ -DENABLE_CACHE_LOCK_DEBUG="${ENABLE_CACHE_LOCK_DEBUG}" \ -DMAKE_TEST=OFF \ + -DBUILD_BENCHMARK="${BUILD_BENCHMARK}" \ -DBUILD_FS_BENCHMARK="${BUILD_FS_BENCHMARK}" \ ${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \ -DWITH_MYSQL="${WITH_MYSQL}" \