Merge branch 'main' into reportqueuestats

Signed-off-by: Anton Pryakhin <[email protected]>

.github/workflows/build.yaml

@@ -13,64 +13,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_dependencies:
-    name: Build deps [ubuntu]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Get dependencies hash
-        id: get-hash
-        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
-      - name: Cache lookup
-        uses: actions/cache/restore@v4
-        id: cache-lookup
-        with:
-          path: deps
-          key: deps-${{ steps.get-hash.outputs.deps_hash }}
-          lookup-only: true
-      - name: Set up dependencies
-        if: steps.cache-lookup.outputs.cache-hit != 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -qy build-essential \
-            gdb \
-            curl \
-            python3.10 \
-            python3-pip \
-            cmake \
-            ninja-build \
-            pkg-config \
-            bison \
-            libfl-dev \
-            libbenchmark-dev \
-            libgmock-dev \
-            libz-dev
-      - name: Fetch & Build non packaged dependencies
-        if: steps.cache-lookup.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p deps
-          cd deps
-          ../docker/build_deps.sh
-      - name: Cache save
-        if: steps.cache-lookup.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
-        with:
-          path: deps
-          key: deps-${{ steps.get-hash.outputs.deps_hash }}
+  get_dependencies:
+    name: "Dependencies"
+    uses: ./.github/workflows/dependencies.yaml
 
   build_ubuntu:
     name: Build [ubuntu]
     runs-on: ubuntu-latest
-    needs: build_dependencies
+    needs: get_dependencies
     steps:
       - uses: actions/checkout@v4
-      - name: Get dependencies hash
-        id: get-hash
-        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
+
       - uses: actions/cache/restore@v4
         with:
           path: deps
-          key: deps-${{ steps.get-hash.outputs.deps_hash }}
+          key: ${{ needs.get_dependencies.outputs.cache_key }}
+
       - name: Set up dependencies
         run: |
           sudo apt-get update
@@ -87,9 +45,11 @@ jobs:
             libbenchmark-dev \
             libgmock-dev \
             libz-dev
+
       - name: Install cached non packaged dependencies
         working-directory: deps
         run: ../docker/build_deps.sh
+
       - name: Build BlazingMQ
         env:
           PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
@@ -103,10 +63,12 @@ jobs:
             -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
             -DCMAKE_INSTALL_LIBDIR=lib64
           cmake --build build/blazingmq --parallel 8 --target all
+
       - name: Clean-up build directories before caching
         run: |
           find . -name "*.o" -type f -delete
           find . -name "*.a" -type f -delete
+
       - uses: actions/cache@v4
         with:
           path: |
@@ -118,16 +80,15 @@ jobs:
   unit_tests_cxx:
     name: UT [c++]
     runs-on: ubuntu-latest
-    needs: build_dependencies
+    needs: get_dependencies
     steps:
       - uses: actions/checkout@v4
-      - name: Get dependencies hash
-        id: get-hash
-        run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
+
       - uses: actions/cache/restore@v4
         with:
           path: deps
-          key: deps-${{ steps.get-hash.outputs.deps_hash }}
+          key: ${{ needs.get_dependencies.outputs.cache_key }}
+
       - name: Set up dependencies
         run: |
           sudo apt-get update
@@ -144,9 +105,11 @@ jobs:
             libbenchmark-dev \
             libgmock-dev \
             libz-dev
+
       - name: Install cached non packaged dependencies
         working-directory: deps
         run: ../docker/build_deps.sh
+
       - name: Build BlazingMQ UTs
         env:
           PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
@@ -160,6 +123,7 @@ jobs:
             -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
             -DCMAKE_INSTALL_LIBDIR=lib64
           cmake --build build/blazingmq --parallel 8 --target all.t
+
       - name: Run C++ Unit Tests
         run: |
           cd ${{ github.workspace }}/build/blazingmq
@@ -171,13 +135,15 @@ jobs:
     needs: build_ubuntu
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/cache/@v4
         with:
           path: |
             build/blazingmq
             deps
             /opt/bb/include
           key: cache-${{ github.sha }}
+
       - name: Run Python Unit Tests
         env:
           PYTHONPATH: ${{ github.workspace }}/src/python
@@ -200,13 +166,15 @@ jobs:
     needs: build_ubuntu
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/cache/@v4
         with:
           path: |
             build/blazingmq
             deps
             /opt/bb/include
           key: cache-${{ github.sha }}
+
       - name: Run Integration Tests
         run: |
           pip install -r ${{ github.workspace }}/src/python/requirements.txt
@@ -239,13 +207,15 @@ jobs:
     needs: build_ubuntu
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/cache/@v4
         with:
           path: |
             build/blazingmq
             deps
             /opt/bb/include
           key: cache-${{ github.sha }}
+
       - name: Run Fuzz Test
         run: |
           pip install -r ${{ github.workspace }}/src/python/requirements.txt
@@ -262,6 +232,7 @@ jobs:
             arch: arm64
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up dependencies
         run: |
           brew install \
@@ -271,6 +242,7 @@ jobs:
             bison \
             flex \
             google-benchmark
+
       - name: Build BlazingMQ
         run: bin/build-darwin.sh
 
@@ -280,13 +252,15 @@ jobs:
     needs: build_ubuntu
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/cache/@v4
         with:
           path: |
             build/blazingmq
             deps
             /opt/bb/include
           key: cache-${{ github.sha }}
+
       - name: Set up plugins dependencies
         run: |
           sudo apt-get update
@@ -304,11 +278,14 @@ jobs:
             libz-dev \
             autoconf \
             libtool
+
       - name: Create dependency fetcher working directory
         run: mkdir -p deps
+
       - name: Fetch & Build non packaged plugins dependencies
         working-directory: deps
         run:  ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh
+
       - name: Build plugins
         env:
           PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
@@ -323,23 +300,30 @@ jobs:
             -DCMAKE_INSTALL_LIBDIR=lib64 \
             -DINSTALL_TARGETS=prometheus
           cmake --build build/blazingmq --parallel 8 --target all
+
       - name: Create prometheus dir
         run:  mkdir -p prometheus_dir
+
       - name: Download Prometheus
         run:  curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/
+
       - name: Run Prometheus
         run:  ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
+
       - name: Run Pushgateway
         run: |
           docker pull prom/pushgateway
           docker run -d -p 9091:9091 prom/pushgateway
+
       - name: Run BMQPrometheus plugin integration test in "pull" mode
         run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker
+
       - name: Clear Prometheus database and restart
         run: |
           curl -X POST "http://localhost:9090/-/quit"
           rm -rf prometheus_dir/data
           ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
+
       - name: Run Prometheus plugin integration test in "push" mode
         run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker
 
@@ -348,13 +332,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -qy doxygen
+
       - name: Set up output directory
         run: |
           mkdir -p docs/docs/apidocs
+
       - name: Build docs
         run: |
           doxygen Doxyfile
@@ -364,6 +351,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
 
@@ -372,7 +360,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Docker Single-Node Workflow
         run: docker compose -f docker/single-node/docker-compose.yaml up --build -d
+
       - name: Docker Cluster Workflow
         run: docker compose -f docker/cluster/docker-compose.yaml up --build -d

.github/workflows/dependencies.yaml

@@ -0,0 +1,70 @@
+name: Dependencies
+
+on:
+  workflow_call:
+    outputs:
+      cache_key:
+        description: "GH Actions Cache key associated with the built dependencies"
+        value: ${{ jobs.build_dependencies.outputs.cache_key }}
+  schedule:
+    # Rebuild dependencies cache from the "main" branch at the beginning of a new day,
+    # so it is accessible from other branches' PRs
+    - cron: "10 0 * * *"
+
+concurrency:
+  group: deps-${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build_dependencies:
+    name: Build [ubuntu]
+    runs-on: ubuntu-latest
+    outputs:
+      cache_key: ${{ steps.get-cache-key.outputs.cache_key }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate dependencies cache key
+        id: get-cache-key
+        run: |
+          echo "cache_key=deps-`date +%y-%m-%d`-`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
+
+      - name: Cache lookup
+        uses: actions/cache/restore@v4
+        id: cache-lookup
+        with:
+          path: deps
+          key: ${{ steps.get-cache-key.outputs.cache_key }}
+          lookup-only: true
+
+      - name: Set up build dependencies
+        if: steps.cache-lookup.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -qy build-essential \
+            gdb \
+            curl \
+            python3.10 \
+            python3-pip \
+            cmake \
+            ninja-build \
+            pkg-config \
+            bison \
+            libfl-dev \
+            libbenchmark-dev \
+            libgmock-dev \
+            libz-dev
+
+      - name: Fetch & build non packaged dependencies
+        if: steps.cache-lookup.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p deps
+          cd deps
+          ../docker/build_deps.sh
+
+      - name: Cache save
+        if: steps.cache-lookup.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: deps
+          key: ${{ steps.get-cache-key.outputs.cache_key }}