From 82fa7528294dd07ca472c83519c8ed0a97335ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=8B=87?= Date: Wed, 10 Apr 2024 14:03:11 +0800 Subject: [PATCH] feat: cmake c++17, rocksdb7.10.2, lz4, zstd, liburing (#232) Co-authored-by: yuzhong.chen --- CMakeLists.txt | 38 +++++++++++++++++++++++++++++++++++--- cmake/liburing.cmake | 40 ++++++++++++++++++++++++++++++++++++++++ cmake/lz4.cmake | 42 ++++++++++++++++++++++++++++++++++++++++++ cmake/rocksdb.cmake | 16 +++++++++++----- cmake/zstd.cmake | 42 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 cmake/liburing.cmake create mode 100644 cmake/lz4.cmake create mode 100644 cmake/zstd.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e7bb40b9..27089f98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -192,6 +192,32 @@ else () include(snappy) endif () +#zstd +if (WITH_SYSTEM_LIBS) + find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + find_library(ZSTD_LIBRARIES names zstd) + if ((NOT ZSTD_INCLUDE_DIR) OR (NOT ZSTD_LIBRARIES)) + message(FATAL_ERROR "Fail to find zstd") + endif () + ADD_LIBRARY(zstd SHARED IMPORTED GLOBAL) +else () + include(zstd) +endif () + +#lz4 +if (WITH_SYSTEM_LIBS) + find_path(LZ4_INCLUDE_DIR NAMES lz4.h) + find_library(LZ4_LIBRARIES names lz4) + if ((NOT LZ4_INCLUDE_DIR) OR (NOT LZ4_LIBRARIES)) + message(FATAL_ERROR "Fail to find lz4") + endif () + ADD_LIBRARY(lz4 SHARED IMPORTED GLOBAL) +else () + include(lz4) +endif () + +include(liburing) + #re2 if (WITH_SYSTEM_LIBS) find_path(RE2_INCLUDE_DIR NAMES re2/re2.h) @@ -333,6 +359,9 @@ SET(DEP_INC ${GFLAGS_INCLUDE_DIR} ${GLOG_INCLUDE_DIR} ${SNAPPY_INCLUDE_DIR} + ${LZ4_INCLUDE_DIR} + ${LIBURING_INCLUDE_DIR} + ${ZSTD_INCLUDE_DIR} ${RE2_INCLUDE_DIR} ${ROCKSDB_INCLUDE_DIR} @@ -363,8 +392,10 @@ SET(DEP_LIB ${GFLAGS_LIBRARIES} ${GLOG_LIBRARIES} ${SNAPPY_LIBRARIES} + ${ZSTD_LIBRARIES} + ${LZ4_LIBRARIES} + ${LIBURING_LIBRARIES} ${RE2_LIBRARIES} - ${ARROW_LIBRARIES} ${FAISS_LIBRARIES} ${OPENBLAS_LIBRARIES} @@ -401,7 +432,7 @@ if (NOT DEBUG) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNDEBUG") endif () -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer ") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer ") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer") add_definitions( @@ -474,7 +505,7 @@ file(GLOB COMMON file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/output/bin) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/output/bin) -set(COMMON_DEPS ssl crypto zlib boost rapidjson arrow openblas faiss croaring glog gflags protobuf re2 snappy rocksdb brpc braft) +set(COMMON_DEPS ssl crypto zlib boost rapidjson arrow openblas faiss croaring glog gflags protobuf re2 snappy lz4 zstd rocksdb brpc braft liburing) if (WITH_GPERF) set(COMMON_DEPS ${COMMON_DEPS} gperf) endif () @@ -483,6 +514,7 @@ add_library(common STATIC ${COMMON} $) add_dependencies(common ${COMMON_DEPS}) target_include_directories(common PUBLIC ${COMMON_INC}) +set(CMAKE_VERBOSE_MAKEFILEON ON) # baikalMeta file(GLOB META_SERVER src/meta_server/*.cpp) set(BAIKALMETA_INC ${COMMON_INC} include/meta_server) diff --git a/cmake/liburing.cmake b/cmake/liburing.cmake new file mode 100644 index 00000000..cdf14aa1 --- /dev/null +++ b/cmake/liburing.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2020-present Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(LIBURING_SOURCES_DIR ${THIRD_PARTY_PATH}/liburing) +SET(LIBURING_INSTALL_DIR ${THIRD_PARTY_PATH}/install/liburing) +SET(LIBURING_INCLUDE_DIR "${LIBURING_INSTALL_DIR}/include" CACHE PATH "liburing include directory." FORCE) +SET(LIBURING_LIBRARIES "${LIBURING_INSTALL_DIR}/lib/liburing.a" CACHE FILEPATH "liburing library." FORCE) + +FILE(WRITE ${LIBURING_SOURCES_DIR}/src/build.sh + "make -j${NUM_OF_PROCESSOR}" + ) + +ExternalProject_Add( + extern_liburing + ${EXTERNAL_PROJECT_LOG_ARGS} + URL "https://github.com/axboe/liburing/archive/refs/tags/liburing-2.4.tar.gz" + PREFIX ${LIBURING_SOURCES_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh + INSTALL_COMMAND mkdir -p ${LIBURING_INSTALL_DIR}/lib/ COMMAND cp ${LIBURING_SOURCES_DIR}/src/extern_liburing/src/liburing.a ${LIBURING_LIBRARIES} COMMAND mkdir -p ${LIBURING_INCLUDE_DIR} COMMAND cp -r ${LIBURING_SOURCES_DIR}/src/extern_liburing/src/include/liburing.h ${LIBURING_SOURCES_DIR}/src/extern_liburing/src/include/liburing ${LIBURING_INCLUDE_DIR} +) + +ADD_LIBRARY(liburing STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET liburing PROPERTY IMPORTED_LOCATION ${LIBURING_LIBRARIES}) +ADD_DEPENDENCIES(liburing extern_liburing) diff --git a/cmake/lz4.cmake b/cmake/lz4.cmake new file mode 100644 index 00000000..66e5846b --- /dev/null +++ b/cmake/lz4.cmake @@ -0,0 +1,42 @@ +# Copyright (c) 2020-present Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(LZ4_SOURCES_DIR ${THIRD_PARTY_PATH}/lz4) +SET(LZ4_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lz4) +SET(LZ4_INCLUDE_DIR "${LZ4_INSTALL_DIR}/include" CACHE PATH "lz4 include directory." FORCE) +SET(LZ4_LIBRARIES "${LZ4_INSTALL_DIR}/lib/liblz4.a" CACHE FILEPATH "lz4 library." FORCE) + +FILE(WRITE ${LZ4_SOURCES_DIR}/src/build.sh + "make -j${NUM_OF_PROCESSOR}" + ) + +ExternalProject_Add( + extern_lz4 + ${EXTERNAL_PROJECT_LOG_ARGS} +# GIT_REPOSITORY "https://github.com/google/lz4.git" +# GIT_TAG "apache-lz4-0.17.1" + URL "https://github.com/lz4/lz4/archive/v1.9.4.tar.gz" + PREFIX ${LZ4_SOURCES_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh + INSTALL_COMMAND mkdir -p ${LZ4_INSTALL_DIR}/lib/ COMMAND cp ${LZ4_SOURCES_DIR}/src/extern_lz4/lib/liblz4.a ${LZ4_LIBRARIES} COMMAND mkdir -p ${LZ4_INCLUDE_DIR} COMMAND cp ${LZ4_SOURCES_DIR}/src/extern_lz4/lib/lz4.h ${LZ4_SOURCES_DIR}/src/extern_lz4/lib/lz4hc.h ${LZ4_INCLUDE_DIR} +) + +ADD_LIBRARY(lz4 STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET lz4 PROPERTY IMPORTED_LOCATION ${LZ4_LIBRARIES}) +ADD_DEPENDENCIES(lz4 extern_lz4) diff --git a/cmake/rocksdb.cmake b/cmake/rocksdb.cmake index c2fc42c3..4994b930 100644 --- a/cmake/rocksdb.cmake +++ b/cmake/rocksdb.cmake @@ -17,9 +17,12 @@ INCLUDE(ExternalProject) SET(ROCKSDB_SOURCES_DIR ${THIRD_PARTY_PATH}/rocksdb) SET(ROCKSDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/rocksdb) SET(ROCKSDB_INCLUDE_DIR "${ROCKSDB_INSTALL_DIR}/include" CACHE PATH "rocksdb include directory." FORCE) -SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/lib/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE) +SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/lib64/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE) +# For some reason, libraries are under either lib or lib64 +include(GNUInstallDirs) +SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE) -set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/gflags") +set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/lz4|${THIRD_PARTY_PATH}/install/zstd|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/liburing") #FILE(WRITE ${ROCKSDB_SOURCES_DIR}/src/build.sh # "PORTABLE=1 make -j${NUM_OF_PROCESSOR} static_lib" @@ -28,9 +31,9 @@ set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/ ExternalProject_Add( extern_rocksdb ${EXTERNAL_PROJECT_LOG_ARGS} - DEPENDS gflags zlib snappy + DEPENDS gflags zlib snappy zstd lz4 liburing PREFIX ${ROCKSDB_SOURCES_DIR} - URL "https://github.com/facebook/rocksdb/archive/v6.29.3.tar.gz" + URL "https://github.com/facebook/rocksdb/archive/v7.10.2.tar.gz" UPDATE_COMMAND "" # CONFIGURE_COMMAND "" # BUILD_IN_SOURCE 1 @@ -50,12 +53,15 @@ ExternalProject_Add( -DCMAKE_PREFIX_PATH=${prefix_path} -DPORTABLE=ON -DWITH_SNAPPY=ON + -DWITH_ZSTD=ON + -DWITH_LZ4=ON -DWITH_RUNTIME_DEBUG=ON -DROCKSDB_BUILD_SHARED=OFF -DWITH_BENCHMARK_TOOLS=OFF -DWITH_CORE_TOOLS=OFF -DWITH_TOOLS=OFF -DUSE_RTTI=ON + -DWITH_LIBURING=ON ${EXTERNAL_OPTIONAL_ARGS} LIST_SEPARATOR | CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ROCKSDB_INSTALL_DIR} @@ -63,7 +69,7 @@ ExternalProject_Add( -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} ) -ADD_DEPENDENCIES(extern_rocksdb zlib snappy gflags) +ADD_DEPENDENCIES(extern_rocksdb zlib snappy zstd lz4 gflags liburing) ADD_LIBRARY(rocksdb STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET rocksdb PROPERTY IMPORTED_LOCATION ${ROCKSDB_LIBRARIES}) ADD_DEPENDENCIES(rocksdb extern_rocksdb) diff --git a/cmake/zstd.cmake b/cmake/zstd.cmake new file mode 100644 index 00000000..e5f28a8c --- /dev/null +++ b/cmake/zstd.cmake @@ -0,0 +1,42 @@ +# Copyright (c) 2020-present Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(ZSTD_SOURCES_DIR ${THIRD_PARTY_PATH}/zstd) +SET(ZSTD_INSTALL_DIR ${THIRD_PARTY_PATH}/install/zstd) +SET(ZSTD_INCLUDE_DIR "${ZSTD_INSTALL_DIR}/include" CACHE PATH "zstd include directory." FORCE) +SET(ZSTD_LIBRARIES "${ZSTD_INSTALL_DIR}/lib/libzstd.a" CACHE FILEPATH "zstd library." FORCE) + +FILE(WRITE ${ZSTD_SOURCES_DIR}/src/build.sh + "cd build/cmake && cmake -DZSTD_LEGACY_SUPPORT=5 -DXXH_NAMESPACE=ZSTD_ && make -j${NUM_OF_PROCESSOR}" + ) + +ExternalProject_Add( + extern_zstd + ${EXTERNAL_PROJECT_LOG_ARGS} +# GIT_REPOSITORY "https://github.com/google/zstd.git" +# GIT_TAG "apache-zstd-0.17.1" + URL "https://github.com/facebook/zstd/archive/v1.4.4.tar.gz" + PREFIX ${ZSTD_SOURCES_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh + INSTALL_COMMAND mkdir -p ${ZSTD_INSTALL_DIR}/lib/ COMMAND cp ${ZSTD_SOURCES_DIR}/src/extern_zstd/build/cmake/lib/libzstd.a ${ZSTD_LIBRARIES} COMMAND mkdir -p ${ZSTD_INCLUDE_DIR} COMMAND cp ${ZSTD_SOURCES_DIR}/src/extern_zstd/lib/zstd.h ${ZSTD_SOURCES_DIR}/src/extern_zstd/lib/dictBuilder/zdict.h ${ZSTD_INCLUDE_DIR} +) + +ADD_LIBRARY(zstd STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET zstd PROPERTY IMPORTED_LOCATION ${ZSTD_LIBRARIES}) +ADD_DEPENDENCIES(zstd extern_zstd)