From 2bc196407c66daea14c090b45e13ece02964cd01 Mon Sep 17 00:00:00 2001 From: zhaochangle Date: Tue, 24 Sep 2024 02:06:57 +0800 Subject: [PATCH 1/2] 1 --- be/CMakeLists.txt | 61 +++++++++++++++---------- be/benchmark/benchmark_main.cpp | 49 ++++++++++++++++++++ be/src/runtime/memory/jemalloc_hook.cpp | 2 +- be/src/service/CMakeLists.txt | 2 +- build.sh | 17 +++++++ 5 files changed, 106 insertions(+), 25 deletions(-) create mode 100644 be/benchmark/benchmark_main.cpp diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 1d79048f96511c..8563abe227cb56 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -130,6 +130,8 @@ message(STATUS "THIRDPARTY_DIR is ${THIRDPARTY_DIR}") option(MAKE_TEST "ON for make unit test or OFF for not" OFF) message(STATUS "make test: ${MAKE_TEST}") +option(BUILD_BENCHMARK "ON for make google benchmark or OFF for not" OFF) +message(STATUS "make benchmark: ${BUILD_BENCHMARK}") option(WITH_MYSQL "Support access MySQL" ON) @@ -568,7 +570,7 @@ if (OS_MACOSX) ) endif() -if (MAKE_TEST) +if (MAKE_TEST OR BUILD_BENCHMARK) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} benchmark @@ -708,6 +710,11 @@ if (MAKE_TEST) endif() endif () +# use this to avoid some runtime tracker. reuse BE_TEST symbol, no need another. +if (BUILD_BENCHMARK) + add_definitions(-DBE_TEST) +endif() + get_directory_property(COMPILER_FLAGS COMPILE_OPTIONS) get_directory_property(COMPILER_DEFINES COMPILE_DEFINITIONS) message(STATUS "Compiler: ${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}") @@ -754,7 +761,7 @@ add_subdirectory(${SRC_DIR}/http) add_subdirectory(${SRC_DIR}/io) add_subdirectory(${SRC_DIR}/olap) add_subdirectory(${SRC_DIR}/runtime) -add_subdirectory(${SRC_DIR}/service) +add_subdirectory(${SRC_DIR}/service) # this include doris_be add_subdirectory(${SRC_DIR}/udf) add_subdirectory(${SRC_DIR}/cloud) @@ -772,36 +779,44 @@ add_subdirectory(${SRC_DIR}/util) add_subdirectory(${SRC_DIR}/vec) add_subdirectory(${SRC_DIR}/pipeline) +# this include doris_be_test if (MAKE_TEST) add_subdirectory(${TEST_DIR}) endif () add_subdirectory(${COMMON_SRC_DIR}/cpp ${BUILD_DIR}/src/common_cpp) -# Install be -install(DIRECTORY DESTINATION ${OUTPUT_DIR}) -install(DIRECTORY DESTINATION ${OUTPUT_DIR}/bin) -install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf) - -install(FILES - ${BASE_DIR}/../bin/start_be.sh - ${BASE_DIR}/../bin/stop_be.sh - ${BASE_DIR}/../tools/jeprof - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE - GROUP_READ GROUP_WRITE GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE - DESTINATION ${OUTPUT_DIR}/bin) - -install(FILES - ${BASE_DIR}/../conf/be.conf - ${BASE_DIR}/../conf/odbcinst.ini - ${BASE_DIR}/../conf/asan_suppr.conf - ${BASE_DIR}/../conf/lsan_suppr.conf - DESTINATION ${OUTPUT_DIR}/conf) +if(NOT BUILD_BENCHMARK) + # Install be + install(DIRECTORY DESTINATION ${OUTPUT_DIR}) + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/bin) + install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf) + + install(FILES + ${BASE_DIR}/../bin/start_be.sh + ${BASE_DIR}/../bin/stop_be.sh + ${BASE_DIR}/../tools/jeprof + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_WRITE GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE + DESTINATION ${OUTPUT_DIR}/bin) + + install(FILES + ${BASE_DIR}/../conf/be.conf + ${BASE_DIR}/../conf/odbcinst.ini + ${BASE_DIR}/../conf/asan_suppr.conf + ${BASE_DIR}/../conf/lsan_suppr.conf + DESTINATION ${OUTPUT_DIR}/conf) +endif() get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) foreach(dir ${dirs}) message(STATUS "dir='${dir}'") endforeach() - +if (BUILD_BENCHMARK) + add_executable(benchmark_test ${BASE_DIR}/benchmark/benchmark_main.cpp) + target_link_libraries(benchmark_test ${DORIS_LINK_LIBS}) + message(STATUS "Add benchmark to build") + install(TARGETS benchmark_test DESTINATION ${OUTPUT_DIR}) +endif() \ No newline at end of file diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp new file mode 100644 index 00000000000000..d3364177e86263 --- /dev/null +++ b/be/benchmark/benchmark_main.cpp @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +#include "vec/columns/column_string.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_string.h" + +namespace doris::vectorized { + +static void Example1(benchmark::State& state) { + state.PauseTiming(); + Block block; + DataTypePtr str_type = std::make_shared(); + std::vector vals {100, "content"}; + state.ResumeTiming(); + + for (auto _ : state) { + auto str_col = ColumnString::create(); + for (auto& v : vals) { + str_col->insert_data(v.data(), v.size()); + } + block.insert({std::move(str_col), str_type, "col"}); + benchmark::DoNotOptimize(block); + } +} +BENCHMARK(Example1); + +} // namespace doris::vectorized + +BENCHMARK_MAIN(); diff --git a/be/src/runtime/memory/jemalloc_hook.cpp b/be/src/runtime/memory/jemalloc_hook.cpp index 445d60d382c270..dffc1344b71dbc 100644 --- a/be/src/runtime/memory/jemalloc_hook.cpp +++ b/be/src/runtime/memory/jemalloc_hook.cpp @@ -60,7 +60,7 @@ void* doris_realloc(void* p, size_t size) __THROW { return nullptr; } -#if USE_MEM_TRACKER +#if defined(USE_MEM_TRACKER) && !defined(BE_TEST) int64_t old_size = jemalloc_usable_size(p); CONSUME_THREAD_MEM_TRACKER_BY_HOOK_WITH_FN( [](size_t size, int64_t old_size) { return jenallocx(size, 0) - old_size; }, size, diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt index 4ce611345840c1..e44045dffce17e 100644 --- a/be/src/service/CMakeLists.txt +++ b/be/src/service/CMakeLists.txt @@ -28,7 +28,7 @@ add_library(Service STATIC ${SRC_FILES}) pch_reuse(Service) -if (${MAKE_TEST} STREQUAL "OFF") +if (${MAKE_TEST} STREQUAL "OFF" AND ${BUILD_BENCHMARK} STREQUAL "OFF") add_executable(doris_be doris_main.cpp ) diff --git a/build.sh b/build.sh index 35c989d0b0a4ec..8cdfd56f95df4b 100755 --- a/build.sh +++ b/build.sh @@ -49,6 +49,7 @@ Usage: $0 --meta-tool build Backend meta tool. Default OFF. --cloud build Cloud. Default OFF. --index-tool build Backend inverted index tool. Default OFF. + --benchmark build Google Benchmark. Default OFF. --broker build Broker. Default ON. --spark-dpp build Spark DPP application. Default ON. --hive-udf build Hive UDF library for Spark Load. Default ON. @@ -64,12 +65,14 @@ Usage: $0 DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default is OFF. DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE. DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not build azure into BE. + Eg. $0 build all $0 --be build Backend $0 --meta-tool build Backend meta tool $0 --cloud build Cloud $0 --index-tool build Backend inverted index tool + $0 --benchmark build Google Benchmark of Backend $0 --fe --clean clean and build Frontend and Spark Dpp application $0 --fe --be --clean clean and build Frontend, Spark Dpp application and Backend $0 --spark-dpp build Spark DPP application alone @@ -129,6 +132,7 @@ if ! OPTS="$(getopt \ -l 'broker' \ -l 'meta-tool' \ -l 'index-tool' \ + -l 'benchmark' \ -l 'spark-dpp' \ -l 'hive-udf' \ -l 'be-java-extensions' \ @@ -151,6 +155,7 @@ BUILD_CLOUD=0 BUILD_BROKER=0 BUILD_META_TOOL='OFF' BUILD_INDEX_TOOL='OFF' +BUILD_BENCHMARK='OFF' BUILD_SPARK_DPP=0 BUILD_BE_JAVA_EXTENSIONS=0 BUILD_HIVE_UDF=0 @@ -170,6 +175,7 @@ if [[ "$#" == 1 ]]; then BUILD_BROKER=1 BUILD_META_TOOL='OFF' BUILD_INDEX_TOOL='OFF' + BUILD_BENCHMARK='OFF' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 BUILD_BE_JAVA_EXTENSIONS=1 @@ -205,6 +211,11 @@ else BUILD_INDEX_TOOL='ON' shift ;; + --benchmark) + BUILD_BENCHMARK='ON' + BUILD_BE=1 # go into BE cmake building, but benchmark instead of doris_be + shift + ;; --spark-dpp) BUILD_SPARK_DPP=1 shift @@ -446,6 +457,10 @@ if [[ -z "${ENABLE_CACHE_LOCK_DEBUG}" ]]; then ENABLE_CACHE_LOCK_DEBUG='OFF' fi +if [[ -z "${BUILD_BENCHMARK}" ]]; then + BUILD_BENCHMARK='OFF' +fi + if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then RECORD_COMPILER_SWITCHES='OFF' fi @@ -476,6 +491,7 @@ echo "Get params: BUILD_BROKER -- ${BUILD_BROKER} BUILD_META_TOOL -- ${BUILD_META_TOOL} BUILD_INDEX_TOOL -- ${BUILD_INDEX_TOOL} + BUILD_BENCHMARK -- ${BUILD_BENCHMARK} BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP} BUILD_BE_JAVA_EXTENSIONS -- ${BUILD_BE_JAVA_EXTENSIONS} BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF} @@ -581,6 +597,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_INJECTION_POINT="${ENABLE_INJECTION_POINT}" \ -DENABLE_CACHE_LOCK_DEBUG="${ENABLE_CACHE_LOCK_DEBUG}" \ -DMAKE_TEST=OFF \ + -DBUILD_BENCHMARK="${BUILD_BENCHMARK}" \ -DBUILD_FS_BENCHMARK="${BUILD_FS_BENCHMARK}" \ ${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \ -DWITH_MYSQL="${WITH_MYSQL}" \ From 5fde914f366733c1301d139c77561371e622d76f Mon Sep 17 00:00:00 2001 From: zhaochangle Date: Mon, 18 Nov 2024 22:08:08 +0800 Subject: [PATCH 2/2] remove old and commit neww --- be/CMakeLists.txt | 4 +- be/benchmark/benchmark_main.cpp | 7 +- be/test/CMakeLists.txt | 11 -- be/test/tools/benchmark_tool.cpp | 320 ------------------------------- bin/start_be.sh | 17 +- build.sh | 4 + 6 files changed, 26 insertions(+), 337 deletions(-) delete mode 100644 be/test/tools/benchmark_tool.cpp diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 8563abe227cb56..d476af8e2110df 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -570,7 +570,7 @@ if (OS_MACOSX) ) endif() -if (MAKE_TEST OR BUILD_BENCHMARK) +if (BUILD_BENCHMARK) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} benchmark @@ -818,5 +818,5 @@ if (BUILD_BENCHMARK) add_executable(benchmark_test ${BASE_DIR}/benchmark/benchmark_main.cpp) target_link_libraries(benchmark_test ${DORIS_LINK_LIBS}) message(STATUS "Add benchmark to build") - install(TARGETS benchmark_test DESTINATION ${OUTPUT_DIR}) + install(TARGETS benchmark_test DESTINATION ${OUTPUT_DIR}/lib) endif() \ No newline at end of file diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp index d3364177e86263..cad6463e981852 100644 --- a/be/benchmark/benchmark_main.cpp +++ b/be/benchmark/benchmark_main.cpp @@ -24,24 +24,27 @@ #include "vec/data_types/data_type.h" #include "vec/data_types/data_type_string.h" -namespace doris::vectorized { +namespace doris::vectorized { // change if need static void Example1(benchmark::State& state) { + // init. dont time it. state.PauseTiming(); Block block; DataTypePtr str_type = std::make_shared(); std::vector vals {100, "content"}; state.ResumeTiming(); + // do test for (auto _ : state) { auto str_col = ColumnString::create(); for (auto& v : vals) { str_col->insert_data(v.data(), v.size()); } block.insert({std::move(str_col), str_type, "col"}); - benchmark::DoNotOptimize(block); + benchmark::DoNotOptimize(block); // mark the watched target } } +// could BENCHMARK many functions to compare them together. BENCHMARK(Example1); } // namespace doris::vectorized diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 5a37267a9379c3..0060d304a859a8 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -113,14 +113,3 @@ if (OS_MACOSX AND ARCH_ARM) COMMAND ${LLVM_STRIP} --strip-all $ ) endif() - -if (BUILD_BENCHMARK_TOOL AND BUILD_BENCHMARK_TOOL STREQUAL "ON") - add_executable(benchmark_tool - tools/benchmark_tool.cpp - testutil/test_util.cpp - olap/tablet_schema_helper.cpp - ) - - target_link_libraries(benchmark_tool ${TEST_LINK_LIBS}) - set_target_properties(benchmark_tool PROPERTIES COMPILE_FLAGS "-fno-access-control") -endif() diff --git a/be/test/tools/benchmark_tool.cpp b/be/test/tools/benchmark_tool.cpp deleted file mode 100644 index 28c8f5c96f5cbe..00000000000000 --- a/be/test/tools/benchmark_tool.cpp +++ /dev/null @@ -1,320 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/compiler_util.h" -#include "common/logging.h" -#include "gutil/strings/split.h" -#include "gutil/strings/substitute.h" -#include "io/fs/file_system.h" -#include "io/fs/file_writer.h" -#include "io/fs/local_file_system.h" -#include "olap/comparison_predicate.h" -#include "olap/data_dir.h" -#include "olap/in_list_predicate.h" -#include "olap/olap_common.h" -#include "olap/row_cursor.h" -#include "olap/rowset/segment_v2/binary_dict_page.h" -#include "olap/rowset/segment_v2/binary_plain_page.h" -#include "olap/rowset/segment_v2/page_builder.h" -#include "olap/rowset/segment_v2/page_decoder.h" -#include "olap/rowset/segment_v2/segment_iterator.h" -#include "olap/rowset/segment_v2/segment_writer.h" -#include "olap/tablet_schema.h" -#include "olap/tablet_schema_helper.h" -#include "olap/types.h" -#include "testutil/test_util.h" -#include "util/debug_util.h" - -DEFINE_string(operation, "Custom", - "valid operation: Custom, BinaryDictPageEncode, BinaryDictPageDecode, SegmentScan, " - "SegmentWrite, " - "SegmentScanByFile, SegmentWriteByFile"); -DEFINE_string(input_file, "./sample.dat", "input file directory"); -DEFINE_string(column_type, "int,varchar", "valid type: int, char, varchar, string"); -DEFINE_string(rows_number, "10000", "rows number"); -DEFINE_string(iterations, "10", - "run times, this is set to 0 means the number of iterations is automatically set "); - -const std::string kSegmentDir = "./segment_benchmark"; - -std::string get_usage(const std::string& progname) { - std::stringstream ss; - ss << progname << " is the Doris BE benchmark tool.\n"; - ss << "Stop BE first before use this tool.\n"; - - ss << "Usage:\n"; - ss << "./benchmark_tool --operation=Custom\n"; - ss << "./benchmark_tool --operation=BinaryDictPageEncode " - "--rows_number=10000 --iterations=40\n"; - ss << "./benchmark_tool --operation=BinaryDictPageDecode " - "--rows_number=10000 --iterations=40\n"; - ss << "./benchmark_tool --operation=SegmentScan --column_type=int,varchar " - "--rows_number=10000 --iterations=0\n"; - ss << "./benchmark_tool --operation=SegmentWrite --column_type=int " - "--rows_number=10000 --iterations=10\n"; - ss << "./benchmark_tool --operation=SegmentScanByFile --input_file=./sample.dat " - "--iterations=10\n"; - ss << "./benchmark_tool --operation=SegmentWriteByFile --input_file=./sample.dat " - "--iterations=10\n"; - - ss << "Sampe data file format: \n" - << "The first line defines Shcema\n" - << "The rest of the content is DataSet\n" - << "For example: \n" - << "int,char,varchar\n" - << "123,hello,world\n" - << "321,good,bye\n"; - return ss.str(); -} - -namespace doris { -class BaseBenchmark { -public: - BaseBenchmark(const std::string& name, int iterations) : _name(name), _iterations(iterations) {} - virtual ~BaseBenchmark() = default; - - void add_name(const std::string& str) { _name += str; } - - virtual void init() {} - virtual void run() {} - - void register_bm() { - auto bm = benchmark::RegisterBenchmark(_name.c_str(), [&](benchmark::State& state) { - //first turn will use more time - this->init(); - this->run(); - for (auto _ : state) { - state.PauseTiming(); - this->init(); - state.ResumeTiming(); - this->run(); - } - }); - if (_iterations != 0) { - bm->Iterations(_iterations); - } - bm->Unit(benchmark::kMillisecond); - } - -private: - std::string _name; - int _iterations; -}; - -class BinaryDictPageBenchmark : public BaseBenchmark { -public: - BinaryDictPageBenchmark(const std::string& name, int iterations) - : BaseBenchmark(name, iterations) {} - virtual ~BinaryDictPageBenchmark() override {} - - virtual void init() override {} - virtual void run() override {} - - void encode_pages(const std::vector& contents) { - PageBuilderOptions options; - BinaryDictPageBuilder page_builder(options); - - results.clear(); - page_start_ids.clear(); - page_start_ids.push_back(0); - for (size_t i = 0; i < contents.size(); i++) { - const Slice* ptr = &contents[i]; - size_t add_num = 1; - (void)page_builder.add(reinterpret_cast(ptr), &add_num); - if (page_builder.is_page_full()) { - OwnedSlice s = page_builder.finish(); - results.emplace_back(std::move(s)); - page_start_ids.push_back(i + 1); - page_builder.reset(); - } - } - OwnedSlice s = page_builder.finish(); - results.emplace_back(std::move(s)); - page_start_ids.push_back(contents.size()); - - (void)page_builder.get_dictionary_page(&dict_slice); - } - - void decode_pages() { - // TODO should rewrite this method by using vectorized next batch method - } - -private: - std::vector results; - OwnedSlice dict_slice; - std::vector page_start_ids; -}; - -class BinaryDictPageEncodeBenchmark : public BinaryDictPageBenchmark { -public: - BinaryDictPageEncodeBenchmark(const std::string& name, int iterations, int rows_number) - : BinaryDictPageBenchmark(name + "/rows_number:" + std::to_string(rows_number), - iterations), - _rows_number(rows_number) {} - virtual ~BinaryDictPageEncodeBenchmark() override {} - - virtual void init() override { - src_strings.clear(); - for (int i = 0; i < _rows_number; i++) { - src_strings.emplace_back(rand_rng_string(rand_rng_int(1, 8))); - } - - slices.clear(); - for (auto s : src_strings) { - slices.emplace_back(s.c_str()); - } - } - virtual void run() override { encode_pages(slices); } - -private: - std::vector slices; - std::vector src_strings; - int _rows_number; -}; - -class BinaryDictPageDecodeBenchmark : public BinaryDictPageBenchmark { -public: - BinaryDictPageDecodeBenchmark(const std::string& name, int iterations, int rows_number) - : BinaryDictPageBenchmark(name + "/rows_number:" + std::to_string(rows_number), - iterations), - _rows_number(rows_number) {} - virtual ~BinaryDictPageDecodeBenchmark() override {} - - virtual void init() override { - src_strings.clear(); - for (int i = 0; i < _rows_number; i++) { - src_strings.emplace_back(rand_rng_string(rand_rng_int(1, 8))); - } - - slices.clear(); - for (auto s : src_strings) { - slices.emplace_back(s.c_str()); - } - - encode_pages(slices); - } - virtual void run() override { decode_pages(); } - -private: - std::vector slices; - std::vector src_strings; - int _rows_number; -}; // namespace doris - -// This is sample custom test. User can write custom test code at custom_init()&custom_run(). -// Call method: ./benchmark_tool --operation=Custom -class CustomBenchmark : public BaseBenchmark { -public: - CustomBenchmark(const std::string& name, int iterations, std::function init_func, - std::function run_func) - : BaseBenchmark(name, iterations), _init_func(init_func), _run_func(run_func) {} - virtual ~CustomBenchmark() override {} - - virtual void init() override { _init_func(); } - virtual void run() override { _run_func(); } - -private: - std::function _init_func; - std::function _run_func; -}; -void custom_init() {} -void custom_run_plus() { - int p = 100000; - int q = 0; - while (p--) { - q++; - if (UNLIKELY(q == 1024)) q = 0; - } -} -void custom_run_mod() { - int p = 100000; - int q = 0; - while (p--) { - q++; - if (q %= 1024) q = 0; - } -} - -class MultiBenchmark { -public: - MultiBenchmark() {} - ~MultiBenchmark() { - for (auto bm : benchmarks) { - delete bm; - } - } - - void add_bm() { - if (equal_ignore_case(FLAGS_operation, "Custom")) { - benchmarks.emplace_back( - new doris::CustomBenchmark("custom_run_plus", std::stoi(FLAGS_iterations), - doris::custom_init, doris::custom_run_plus)); - benchmarks.emplace_back( - new doris::CustomBenchmark("custom_run_mod", std::stoi(FLAGS_iterations), - doris::custom_init, doris::custom_run_mod)); - } else if (equal_ignore_case(FLAGS_operation, "BinaryDictPageEncode")) { - benchmarks.emplace_back(new doris::BinaryDictPageEncodeBenchmark( - FLAGS_operation, std::stoi(FLAGS_iterations), std::stoi(FLAGS_rows_number))); - } else if (equal_ignore_case(FLAGS_operation, "BinaryDictPageDecode")) { - benchmarks.emplace_back(new doris::BinaryDictPageDecodeBenchmark( - FLAGS_operation, std::stoi(FLAGS_iterations), std::stoi(FLAGS_rows_number))); - } else { - std::cout << "operation invalid!" << std::endl; - } - } - void register_bm() { - for (auto bm : benchmarks) { - bm->register_bm(); - } - } - -private: - std::vector benchmarks; -}; - -} //namespace doris -int main(int argc, char** argv) { - std::string usage = get_usage(argv[0]); - gflags::SetUsageMessage(usage); - google::ParseCommandLineFlags(&argc, &argv, true); - - doris::StoragePageCache::create_global_cache(1 << 30, 10, 0); - - doris::MultiBenchmark multi_bm; - multi_bm.add_bm(); - multi_bm.register_bm(); - - benchmark::Initialize(&argc, argv); - benchmark::RunSpecifiedBenchmarks(); - benchmark::Shutdown(); - - return 0; -} diff --git a/bin/start_be.sh b/bin/start_be.sh index 0ae0914d42ec20..5b7df5ead589fa 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -32,6 +32,7 @@ OPTS="$(getopt \ -l 'daemon' \ -l 'console' \ -l 'version' \ + -l 'benchmark' \ -- "$@")" eval set -- "${OPTS}" @@ -39,6 +40,8 @@ eval set -- "${OPTS}" RUN_DAEMON=0 RUN_CONSOLE=0 RUN_VERSION=0 +RUN_BENCHMARK=0 + while true; do case "$1" in --daemon) @@ -53,6 +56,10 @@ while true; do RUN_VERSION=1 shift ;; + --benchmark) + RUN_BENCHMARK=1 + shift + ;; --) shift break @@ -293,7 +300,7 @@ fi pidfile="${PID_DIR}/be.pid" -if [[ -f "${pidfile}" ]]; then +if [[ -f "${pidfile}" && "${RUN_BENCHMARK}" -eq 0 ]]; then if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then echo "Backend is already running as process $(cat "${pidfile}"), stop it first" exit 1 @@ -419,7 +426,13 @@ else export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" fi -if [[ "${RUN_DAEMON}" -eq 1 ]]; then +if [[ "${RUN_BENCHMARK}" -eq 1 ]]; then + if [[ "$(uname -s)" == 'Darwin' ]]; then + env DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}" ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/benchmark_test" + else + ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/benchmark_test" + fi +elif [[ "${RUN_DAEMON}" -eq 1 ]]; then if [[ "$(uname -s)" == 'Darwin' ]]; then nohup env DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}" ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1