Perf[MQB]: make independent item pools for channels #2309
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| build_dependencies: | |
| name: Build deps [ubuntu] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get dependencies hash | |
| id: get-hash | |
| run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT | |
| - name: Cache lookup | |
| uses: actions/cache/restore@v4 | |
| id: cache-lookup | |
| with: | |
| path: deps | |
| key: deps-${{ steps.get-hash.outputs.deps_hash }} | |
| lookup-only: true | |
| - name: Set up dependencies | |
| if: steps.cache-lookup.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| python3-pip \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev | |
| - name: Fetch & Build non packaged dependencies | |
| if: steps.cache-lookup.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p deps | |
| cd deps | |
| ../docker/build_deps.sh | |
| - name: Cache save | |
| if: steps.cache-lookup.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: deps | |
| key: deps-${{ steps.get-hash.outputs.deps_hash }} | |
| build_ubuntu: | |
| name: Build [ubuntu] | |
| runs-on: ubuntu-latest | |
| needs: build_dependencies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get dependencies hash | |
| id: get-hash | |
| run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT | |
| - uses: actions/cache/restore@v4 | |
| with: | |
| path: deps | |
| key: deps-${{ steps.get-hash.outputs.deps_hash }} | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| python3-pip \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev | |
| - name: Install cached non packaged dependencies | |
| working-directory: deps | |
| run: ../docker/build_deps.sh | |
| - name: Build BlazingMQ | |
| env: | |
| PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig | |
| run: | | |
| cmake -S . -B build/blazingmq -G Ninja \ | |
| -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DBDE_BUILD_TARGET_SAFE=ON \ | |
| -DBDE_BUILD_TARGET_64=ON \ | |
| -DBDE_BUILD_TARGET_CPP17=ON \ | |
| -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ | |
| -DCMAKE_INSTALL_LIBDIR=lib64 | |
| cmake --build build/blazingmq --parallel 8 --target all | |
| - name: Clean-up build directories before caching | |
| run: | | |
| find . -name "*.o" -type f -delete | |
| find . -name "*.a" -type f -delete | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| unit_tests_cxx: | |
| name: UT [c++] | |
| runs-on: ubuntu-latest | |
| needs: build_dependencies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get dependencies hash | |
| id: get-hash | |
| run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT | |
| - uses: actions/cache/restore@v4 | |
| with: | |
| path: deps | |
| key: deps-${{ steps.get-hash.outputs.deps_hash }} | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| python3-pip \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev | |
| - name: Install cached non packaged dependencies | |
| working-directory: deps | |
| run: ../docker/build_deps.sh | |
| - name: Build BlazingMQ UTs | |
| env: | |
| PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig | |
| run: | | |
| cmake -S . -B build/blazingmq -G Ninja \ | |
| -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DBDE_BUILD_TARGET_SAFE=ON \ | |
| -DBDE_BUILD_TARGET_64=ON \ | |
| -DBDE_BUILD_TARGET_CPP17=ON \ | |
| -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ | |
| -DCMAKE_INSTALL_LIBDIR=lib64 | |
| cmake --build build/blazingmq --parallel 8 --target all.t | |
| - name: Run C++ Unit Tests | |
| run: | | |
| cd ${{ github.workspace }}/build/blazingmq | |
| ctest -E bmqsys_executil.t --output-on-failure | |
| unit_tests_python: | |
| name: UT [python] | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Run Python Unit Tests | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/src/python | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/python/requirements.txt | |
| pip install "xsdata[cli]" | |
| cd ${{ github.workspace }} | |
| src/python/bin/schemagen | |
| src/python/bin/tweakgen | |
| pytest src/python | |
| integration_tests_ubuntu: | |
| name: IT [${{ matrix.cluster }}/${{ matrix.mode }}] | |
| strategy: | |
| matrix: | |
| mode: ["legacy_mode", "fsm_mode"] | |
| cluster: ["single", "multi"] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Run Integration Tests | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/python/requirements.txt | |
| ${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \ | |
| --log-level ERROR \ | |
| --log-file-level=info \ | |
| --bmq-tolerate-dirty-shutdown \ | |
| --bmq-log-dir=failure-logs \ | |
| --bmq-log-level=INFO \ | |
| --junitxml=integration-tests.xml \ | |
| --tb long \ | |
| --reruns=3 \ | |
| -n 4 -v | |
| fuzz_tests_ubuntu: | |
| name: Fuzz test [${{ matrix.request }}] | |
| strategy: | |
| matrix: | |
| request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Run Fuzz Test | |
| run: | | |
| pip install -r ${{ github.workspace }}/src/python/requirements.txt | |
| cd src/python | |
| python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }} | |
| build_macosx: | |
| name: Build [macosx_${{ matrix.arch }}] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: macos-14 | |
| arch: arm64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up dependencies | |
| run: | | |
| brew install \ | |
| curl \ | |
| [email protected] \ | |
| ninja \ | |
| bison \ | |
| flex \ | |
| google-benchmark | |
| - name: Build BlazingMQ | |
| run: bin/build-darwin.sh | |
| build_and_test_bmqprometheus_plugin: | |
| name: "Build Prometheus plugin [ubuntu]" | |
| runs-on: ubuntu-latest | |
| needs: build_ubuntu | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/cache/@v4 | |
| with: | |
| path: | | |
| build/blazingmq | |
| deps | |
| /opt/bb/include | |
| key: cache-${{ github.sha }} | |
| - name: Set up plugins dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy build-essential \ | |
| gdb \ | |
| curl \ | |
| python3.10 \ | |
| cmake \ | |
| ninja-build \ | |
| pkg-config \ | |
| bison \ | |
| libfl-dev \ | |
| libbenchmark-dev \ | |
| libgmock-dev \ | |
| libz-dev \ | |
| autoconf \ | |
| libtool | |
| - name: Create dependency fetcher working directory | |
| run: mkdir -p deps | |
| - name: Fetch & Build non packaged plugins dependencies | |
| working-directory: deps | |
| run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh | |
| - name: Build plugins | |
| env: | |
| PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig | |
| run: | | |
| cmake -S . -B build/blazingmq -G Ninja \ | |
| -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DBDE_BUILD_TARGET_SAFE=ON \ | |
| -DBDE_BUILD_TARGET_64=ON \ | |
| -DBDE_BUILD_TARGET_CPP17=ON \ | |
| -DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \ | |
| -DCMAKE_INSTALL_LIBDIR=lib64 \ | |
| -DINSTALL_TARGETS=prometheus | |
| cmake --build build/blazingmq --parallel 8 --target all | |
| - name: Create prometheus dir | |
| run: mkdir -p prometheus_dir | |
| - name: Download Prometheus | |
| run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/ | |
| - name: Run Prometheus | |
| run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & | |
| - name: Run Pushgateway | |
| run: | | |
| docker pull prom/pushgateway | |
| docker run -d -p 9091:9091 prom/pushgateway | |
| - name: Run BMQPrometheus plugin integration test in "pull" mode | |
| run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker | |
| - name: Clear Prometheus database and restart | |
| run: | | |
| curl -X POST "http://localhost:9090/-/quit" | |
| rm -rf prometheus_dir/data | |
| ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data & | |
| - name: Run Prometheus plugin integration test in "push" mode | |
| run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker | |
| documentation: | |
| name: "Documentation" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -qy doxygen | |
| - name: Build docs | |
| run: | | |
| doxygen Doxyfile | |
| shellcheck: | |
| name: Shellcheck | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run ShellCheck | |
| uses: ludeeus/action-shellcheck@master | |
| docker_build_ubuntu: | |
| name: "Docker [ubuntu]" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Docker Single-Node Workflow | |
| run: docker compose -f docker/single-node/docker-compose.yaml up --build -d | |
| - name: Docker Cluster Workflow | |
| run: docker compose -f docker/cluster/docker-compose.yaml up --build -d |