Skip to content

Commit

Permalink
feat: cmake c++17, rocksdb7.10.2, lz4, zstd, liburing
Browse files Browse the repository at this point in the history
  • Loading branch information
cyz-2023 committed Apr 9, 2024
1 parent 844a6d1 commit e0afbee
Show file tree
Hide file tree
Showing 5 changed files with 167 additions and 8 deletions.
38 changes: 35 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,32 @@ else ()
include(snappy)
endif ()

#zstd
if (WITH_SYSTEM_LIBS)
find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
find_library(ZSTD_LIBRARIES names zstd)
if ((NOT ZSTD_INCLUDE_DIR) OR (NOT ZSTD_LIBRARIES))
message(FATAL_ERROR "Fail to find zstd")
endif ()
ADD_LIBRARY(zstd SHARED IMPORTED GLOBAL)
else ()
include(zstd)
endif ()

#lz4
if (WITH_SYSTEM_LIBS)
find_path(LZ4_INCLUDE_DIR NAMES lz4.h)
find_library(LZ4_LIBRARIES names lz4)
if ((NOT LZ4_INCLUDE_DIR) OR (NOT LZ4_LIBRARIES))
message(FATAL_ERROR "Fail to find lz4")
endif ()
ADD_LIBRARY(lz4 SHARED IMPORTED GLOBAL)
else ()
include(lz4)
endif ()

include(liburing)

#re2
if (WITH_SYSTEM_LIBS)
find_path(RE2_INCLUDE_DIR NAMES re2/re2.h)
Expand Down Expand Up @@ -333,6 +359,9 @@ SET(DEP_INC
${GFLAGS_INCLUDE_DIR}
${GLOG_INCLUDE_DIR}
${SNAPPY_INCLUDE_DIR}
${LZ4_INCLUDE_DIR}
${LIBURING_INCLUDE_DIR}
${ZSTD_INCLUDE_DIR}
${RE2_INCLUDE_DIR}

${ROCKSDB_INCLUDE_DIR}
Expand Down Expand Up @@ -363,8 +392,10 @@ SET(DEP_LIB
${GFLAGS_LIBRARIES}
${GLOG_LIBRARIES}
${SNAPPY_LIBRARIES}
${ZSTD_LIBRARIES}
${LZ4_LIBRARIES}
${LIBURING_LIBRARIES}
${RE2_LIBRARIES}

${ARROW_LIBRARIES}
${FAISS_LIBRARIES}
${OPENBLAS_LIBRARIES}
Expand Down Expand Up @@ -401,7 +432,7 @@ if (NOT DEBUG)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNDEBUG")
endif ()

SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer ")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer ")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 ${DEBUG_SYMBOL} -O2 -pipe -m64 -fopenmp -Wall -W -fPIC -Wno-unused-parameter -Wno-strict-aliasing -Wno-parentheses -fno-omit-frame-pointer")

add_definitions(
Expand Down Expand Up @@ -474,7 +505,7 @@ file(GLOB COMMON
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/output/bin)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/output/bin)

set(COMMON_DEPS ssl crypto zlib boost rapidjson arrow openblas faiss croaring glog gflags protobuf re2 snappy rocksdb brpc braft)
set(COMMON_DEPS ssl crypto zlib boost rapidjson arrow openblas faiss croaring glog gflags protobuf re2 snappy lz4 zstd rocksdb brpc braft liburing)
if (WITH_GPERF)
set(COMMON_DEPS ${COMMON_DEPS} gperf)
endif ()
Expand All @@ -483,6 +514,7 @@ add_library(common STATIC ${COMMON} $<TARGET_OBJECTS:PROTO_OBJS>)
add_dependencies(common ${COMMON_DEPS})
target_include_directories(common PUBLIC ${COMMON_INC})

set(CMAKE_VERBOSE_MAKEFILEON ON)
# baikalMeta
file(GLOB META_SERVER src/meta_server/*.cpp)
set(BAIKALMETA_INC ${COMMON_INC} include/meta_server)
Expand Down
40 changes: 40 additions & 0 deletions cmake/liburing.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Copyright (c) 2020-present Baidu, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

INCLUDE(ExternalProject)

SET(LIBURING_SOURCES_DIR ${THIRD_PARTY_PATH}/liburing)
SET(LIBURING_INSTALL_DIR ${THIRD_PARTY_PATH}/install/liburing)
SET(LIBURING_INCLUDE_DIR "${LIBURING_INSTALL_DIR}/include" CACHE PATH "liburing include directory." FORCE)
SET(LIBURING_LIBRARIES "${LIBURING_INSTALL_DIR}/lib/liburing.a" CACHE FILEPATH "liburing library." FORCE)

FILE(WRITE ${LIBURING_SOURCES_DIR}/src/build.sh
"make -j${NUM_OF_PROCESSOR}"
)

ExternalProject_Add(
extern_liburing
${EXTERNAL_PROJECT_LOG_ARGS}
URL "https://github.com/axboe/liburing/archive/refs/tags/liburing-2.4.tar.gz"
PREFIX ${LIBURING_SOURCES_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_IN_SOURCE 1
BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh
INSTALL_COMMAND mkdir -p ${LIBURING_INSTALL_DIR}/lib/ COMMAND cp ${LIBURING_SOURCES_DIR}/src/extern_liburing/src/liburing.a ${LIBURING_LIBRARIES} COMMAND mkdir -p ${LIBURING_INCLUDE_DIR} COMMAND cp -r ${LIBURING_SOURCES_DIR}/src/extern_liburing/src/include/liburing.h ${LIBURING_SOURCES_DIR}/src/extern_liburing/src/include/liburing ${LIBURING_INCLUDE_DIR}
)

ADD_LIBRARY(liburing STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET liburing PROPERTY IMPORTED_LOCATION ${LIBURING_LIBRARIES})
ADD_DEPENDENCIES(liburing extern_liburing)
42 changes: 42 additions & 0 deletions cmake/lz4.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Copyright (c) 2020-present Baidu, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

INCLUDE(ExternalProject)

SET(LZ4_SOURCES_DIR ${THIRD_PARTY_PATH}/lz4)
SET(LZ4_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lz4)
SET(LZ4_INCLUDE_DIR "${LZ4_INSTALL_DIR}/include" CACHE PATH "lz4 include directory." FORCE)
SET(LZ4_LIBRARIES "${LZ4_INSTALL_DIR}/lib/liblz4.a" CACHE FILEPATH "lz4 library." FORCE)

FILE(WRITE ${LZ4_SOURCES_DIR}/src/build.sh
"make -j${NUM_OF_PROCESSOR}"
)

ExternalProject_Add(
extern_lz4
${EXTERNAL_PROJECT_LOG_ARGS}
# GIT_REPOSITORY "https://github.com/google/lz4.git"
# GIT_TAG "apache-lz4-0.17.1"
URL "https://github.com/lz4/lz4/archive/v1.9.4.tar.gz"
PREFIX ${LZ4_SOURCES_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_IN_SOURCE 1
BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh
INSTALL_COMMAND mkdir -p ${LZ4_INSTALL_DIR}/lib/ COMMAND cp ${LZ4_SOURCES_DIR}/src/extern_lz4/lib/liblz4.a ${LZ4_LIBRARIES} COMMAND mkdir -p ${LZ4_INCLUDE_DIR} COMMAND cp ${LZ4_SOURCES_DIR}/src/extern_lz4/lib/lz4.h ${LZ4_SOURCES_DIR}/src/extern_lz4/lib/lz4hc.h ${LZ4_INCLUDE_DIR}
)

ADD_LIBRARY(lz4 STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET lz4 PROPERTY IMPORTED_LOCATION ${LZ4_LIBRARIES})
ADD_DEPENDENCIES(lz4 extern_lz4)
13 changes: 8 additions & 5 deletions cmake/rocksdb.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ INCLUDE(ExternalProject)
SET(ROCKSDB_SOURCES_DIR ${THIRD_PARTY_PATH}/rocksdb)
SET(ROCKSDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/rocksdb)
SET(ROCKSDB_INCLUDE_DIR "${ROCKSDB_INSTALL_DIR}/include" CACHE PATH "rocksdb include directory." FORCE)
SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/lib/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE)
SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/lib64/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE)

set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/gflags")
set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/lz4|${THIRD_PARTY_PATH}/install/zstd|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/liburing")

#FILE(WRITE ${ROCKSDB_SOURCES_DIR}/src/build.sh
# "PORTABLE=1 make -j${NUM_OF_PROCESSOR} static_lib"
Expand All @@ -28,9 +28,9 @@ set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/
ExternalProject_Add(
extern_rocksdb
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS gflags zlib snappy
DEPENDS gflags zlib snappy zstd lz4 liburing
PREFIX ${ROCKSDB_SOURCES_DIR}
URL "https://github.com/facebook/rocksdb/archive/v6.29.3.tar.gz"
URL "https://github.com/facebook/rocksdb/archive/v7.10.2.tar.gz"
UPDATE_COMMAND ""
# CONFIGURE_COMMAND ""
# BUILD_IN_SOURCE 1
Expand All @@ -50,20 +50,23 @@ ExternalProject_Add(
-DCMAKE_PREFIX_PATH=${prefix_path}
-DPORTABLE=ON
-DWITH_SNAPPY=ON
-DWITH_ZSTD=ON
-DWITH_LZ4=ON
-DWITH_RUNTIME_DEBUG=ON
-DROCKSDB_BUILD_SHARED=OFF
-DWITH_BENCHMARK_TOOLS=OFF
-DWITH_CORE_TOOLS=OFF
-DWITH_TOOLS=OFF
-DUSE_RTTI=ON
-DWITH_LIBURING=ON
${EXTERNAL_OPTIONAL_ARGS}
LIST_SEPARATOR |
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ROCKSDB_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)

ADD_DEPENDENCIES(extern_rocksdb zlib snappy gflags)
ADD_DEPENDENCIES(extern_rocksdb zlib snappy zstd lz4 gflags liburing)
ADD_LIBRARY(rocksdb STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET rocksdb PROPERTY IMPORTED_LOCATION ${ROCKSDB_LIBRARIES})
ADD_DEPENDENCIES(rocksdb extern_rocksdb)
42 changes: 42 additions & 0 deletions cmake/zstd.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Copyright (c) 2020-present Baidu, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

INCLUDE(ExternalProject)

SET(ZSTD_SOURCES_DIR ${THIRD_PARTY_PATH}/zstd)
SET(ZSTD_INSTALL_DIR ${THIRD_PARTY_PATH}/install/zstd)
SET(ZSTD_INCLUDE_DIR "${ZSTD_INSTALL_DIR}/include" CACHE PATH "zstd include directory." FORCE)
SET(ZSTD_LIBRARIES "${ZSTD_INSTALL_DIR}/lib/libzstd.a" CACHE FILEPATH "zstd library." FORCE)

FILE(WRITE ${ZSTD_SOURCES_DIR}/src/build.sh
"cd build/cmake && cmake -DZSTD_LEGACY_SUPPORT=5 -DXXH_NAMESPACE=ZSTD_ && make -j${NUM_OF_PROCESSOR}"
)

ExternalProject_Add(
extern_zstd
${EXTERNAL_PROJECT_LOG_ARGS}
# GIT_REPOSITORY "https://github.com/google/zstd.git"
# GIT_TAG "apache-zstd-0.17.1"
URL "https://github.com/facebook/zstd/archive/v1.4.4.tar.gz"
PREFIX ${ZSTD_SOURCES_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_IN_SOURCE 1
BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh
INSTALL_COMMAND mkdir -p ${ZSTD_INSTALL_DIR}/lib/ COMMAND cp ${ZSTD_SOURCES_DIR}/src/extern_zstd/build/cmake/lib/libzstd.a ${ZSTD_LIBRARIES} COMMAND mkdir -p ${ZSTD_INCLUDE_DIR} COMMAND cp ${ZSTD_SOURCES_DIR}/src/extern_zstd/lib/zstd.h ${ZSTD_SOURCES_DIR}/src/extern_zstd/lib/dictBuilder/zdict.h ${ZSTD_INCLUDE_DIR}
)

ADD_LIBRARY(zstd STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET zstd PROPERTY IMPORTED_LOCATION ${ZSTD_LIBRARIES})
ADD_DEPENDENCIES(zstd extern_zstd)

0 comments on commit e0afbee

Please sign in to comment.