[feature](merge-cloud) Publish meta service and recycler service (apa…

…che#30054) Co-authored-by: plat1ko <[email protected]> Co-authored-by: Gavin Chou <[email protected]> Co-authored-by: Xiaocc <[email protected]> Co-authored-by: deardeng <[email protected]> Co-authored-by: meiyi <[email protected]> Co-authored-by: Luwei <[email protected]> Co-authored-by: Xin Liao <[email protected]> Co-authored-by: Lei Zhang <[email protected]> Co-authored-by: Lightman <[email protected]> Co-authored-by: YueW <[email protected]> Co-authored-by: HHoflittlefish777 <[email protected]> Co-authored-by: Siyang Tang <[email protected]> Co-authored-by: bobhan1 <[email protected]> Co-authored-by: Kidd <[email protected]> Co-authored-by: Tanya-W <[email protected]> Co-authored-by: panDing19 <[email protected]> Co-authored-by: AlexYue <[email protected]> Co-authored-by: abmdocrt <[email protected]>
rohitrs1983 · Jan 18, 2024 · 6ab8a07 · 6ab8a07
1 parent 4f4e207
commit 6ab8a07
Show file tree

Hide file tree

Showing 117 changed files with 49,361 additions and 8 deletions.
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -79,15 +79,16 @@ jobs:
         id: filter
         with:
           filters: |
-            be_changes:
+            cpp_changes:
               - 'be/**'
+              - 'cloud/**'
               - 'gensrc/proto/**'
               - 'gensrc/thrift/**'
 
       - name: Generate compile_commands.json
         id: generate
         run: |
-          if [[ "${{ steps.filter.outputs.be_changes }}" == 'true' ]]; then
+          if [[ "${{ steps.filter.outputs.cpp_changes }}" == 'true' ]]; then
             export DEFAULT_DIR='/opt/doris'
 
             mkdir "${DEFAULT_DIR}"
@@ -104,14 +105,14 @@ jobs:
             popd
 
             export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be
+            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be --cloud
           fi
 
-          echo "should_check=${{ steps.filter.outputs.be_changes }}" >>${GITHUB_OUTPUT}
+          echo "should_check=${{ steps.filter.outputs.cpp_changes }}" >>${GITHUB_OUTPUT}
 
       - name: Upload
         uses: actions/upload-artifact@v3
-        if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        if: ${{ steps.filter.outputs.cpp_changes == 'true' }}
         with:
           name: compile_commands
           path: ./be/build_Release/compile_commands.json

diff --git a/.gitignore b/.gitignore
@@ -99,6 +99,11 @@ be/.devcontainer/
 be/src/apache-orc/
 zoneinfo/
 
+# Cloud
+cloud/build*/
+cloud/cmake-build*/
+cloud/ut_build*/
+
 ## tools
 tools/ssb-tools/ssb-data/
 tools/ssb-tools/ssb-dbgen/

diff --git a/build.sh b/build.sh
@@ -42,6 +42,7 @@ Usage: $0 <options>
      --fe                   build Frontend and Spark DPP application. Default ON.
      --be                   build Backend. Default ON.
      --meta-tool            build Backend meta tool. Default OFF.
+     --cloud                build Cloud. Default OFF.
      --index-tool           build Backend inverted index tool. Default OFF.
      --broker               build Broker. Default ON.
      --audit                build audit loader. Default ON.
@@ -61,6 +62,7 @@ Usage: $0 <options>
     $0                                      build all
     $0 --be                                 build Backend
     $0 --meta-tool                          build Backend meta tool
+    $0 --cloud                              build Cloud
     $0 --index-tool                         build Backend inverted index tool
     $0 --fe --clean                         clean and build Frontend and Spark Dpp application
     $0 --fe --be --clean                    clean and build Frontend, Spark Dpp application and Backend
@@ -117,6 +119,7 @@ if ! OPTS="$(getopt \
     -o '' \
     -l 'fe' \
     -l 'be' \
+    -l 'cloud' \
     -l 'broker' \
     -l 'audit' \
     -l 'meta-tool' \
@@ -138,6 +141,7 @@ eval set -- "${OPTS}"
 PARALLEL="$(($(nproc) / 4 + 1))"
 BUILD_FE=0
 BUILD_BE=0
+BUILD_CLOUD=0
 BUILD_BROKER=0
 BUILD_AUDIT=0
 BUILD_META_TOOL='OFF'
@@ -154,6 +158,7 @@ if [[ "$#" == 1 ]]; then
     # default
     BUILD_FE=1
     BUILD_BE=1
+
     BUILD_BROKER=1
     BUILD_AUDIT=1
     BUILD_META_TOOL='OFF'
@@ -177,6 +182,10 @@ else
             BUILD_BE_JAVA_EXTENSIONS=1
             shift
             ;;
+        --cloud)
+            BUILD_CLOUD=1
+            shift
+            ;;
         --broker)
             BUILD_BROKER=1
             shift
@@ -244,6 +253,7 @@ else
     if [[ "${PARAMETER_COUNT}" -eq 3 ]] && [[ "${PARAMETER_FLAG}" -eq 1 ]]; then
         BUILD_FE=1
         BUILD_BE=1
+        BUILD_CLOUD=1
         BUILD_BROKER=1
         BUILD_AUDIT=1
         BUILD_META_TOOL='ON'
@@ -307,7 +317,7 @@ update_submodule() {
     fi
 }
 
-if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 ]]; then
+if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 && ${BUILD_CLOUD} -eq 0 ]]; then
     clean_gensrc
     clean_be
     clean_fe
@@ -415,6 +425,7 @@ fi
 echo "Get params:
     BUILD_FE                    -- ${BUILD_FE}
     BUILD_BE                    -- ${BUILD_BE}
+    BUILD_CLOUD                 -- ${BUILD_CLOUD}
     BUILD_BROKER                -- ${BUILD_BROKER}
     BUILD_AUDIT                 -- ${BUILD_AUDIT}
     BUILD_META_TOOL             -- ${BUILD_META_TOOL}
@@ -541,6 +552,42 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
     cd "${DORIS_HOME}"
 fi
 
+# Clean and build cloud
+if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
+    if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/cloud_version.h" ]]; then
+        rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/cloud_version.h"
+    fi
+    CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
+    echo "Build Cloud: ${CMAKE_BUILD_TYPE}"
+    CMAKE_BUILD_DIR="${DORIS_HOME}/cloud/build_${CMAKE_BUILD_TYPE}"
+    if [[ "${CLEAN}" -eq 1 ]]; then
+        rm -rf "${CMAKE_BUILD_DIR}"
+        echo "clean cloud"
+    fi
+    MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")"
+    echo "-- Make program: ${MAKE_PROGRAM}"
+    echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}"
+    mkdir -p "${CMAKE_BUILD_DIR}"
+    cd "${CMAKE_BUILD_DIR}"
+    "${CMAKE_CMD}" -G "${GENERATOR}" \
+        -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \
+        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+        -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
+        -DMAKE_TEST=OFF \
+        "${CMAKE_USE_CCACHE}" \
+        -DUSE_LIBCPP="${USE_LIBCPP}" \
+        -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
+        -DUSE_DWARF="${USE_DWARF}" \
+        -DUSE_JEMALLOC="${USE_JEMALLOC}" \
+        -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
+        -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
+        "${DORIS_HOME}/cloud/"
+    "${BUILD_SYSTEM}" -j "${PARALLEL}"
+    "${BUILD_SYSTEM}" install
+    cd "${DORIS_HOME}"
+    echo "Build cloud done"
+fi
+
 if [[ "${BUILD_DOCS}" = "ON" ]]; then
     # Build docs, should be built before Frontend
     echo "Build docs"
@@ -754,6 +801,11 @@ if [[ "${BUILD_AUDIT}" -eq 1 ]]; then
     cd "${DORIS_HOME}"
 fi
 
+if [[ ${BUILD_CLOUD} -eq 1 ]]; then
+    rm -rf "${DORIS_HOME}/output/ms"
+    cp -r -p "${DORIS_HOME}/cloud/output" "${DORIS_HOME}/output/ms"
+fi
+
 echo "***************************************"
 echo "Successfully build Doris"
 echo "***************************************"