Skip to content

Perf[MQB]: make independent item pools for channels #2295

Perf[MQB]: make independent item pools for channels

Perf[MQB]: make independent item pools for channels #2295

Workflow file for this run

name: Tests
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
build_dependencies:
name: Build deps [ubuntu]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Get dependencies hash
id: get-hash
run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
- name: Cache lookup
uses: actions/cache/restore@v4
id: cache-lookup
with:
path: deps
key: deps-${{ steps.get-hash.outputs.deps_hash }}
lookup-only: true
- name: Set up dependencies
if: steps.cache-lookup.outputs.cache-hit != 'true'
run: |
sudo apt-get update
sudo apt-get install -qy build-essential \
gdb \
curl \
python3.10 \
python3-pip \
cmake \
ninja-build \
pkg-config \
bison \
libfl-dev \
libbenchmark-dev \
libgmock-dev \
libz-dev
- name: Fetch & Build non packaged dependencies
if: steps.cache-lookup.outputs.cache-hit != 'true'
run: |
mkdir -p deps
cd deps
../docker/build_deps.sh
- name: Cache save
if: steps.cache-lookup.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: deps
key: deps-${{ steps.get-hash.outputs.deps_hash }}
build_ubuntu:
name: Build [ubuntu]
runs-on: ubuntu-latest
needs: build_dependencies
steps:
- uses: actions/checkout@v4
- name: Get dependencies hash
id: get-hash
run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
- uses: actions/cache/restore@v4
with:
path: deps
key: deps-${{ steps.get-hash.outputs.deps_hash }}
- name: Set up dependencies
run: |
sudo apt-get update
sudo apt-get install -qy build-essential \
gdb \
curl \
python3.10 \
python3-pip \
cmake \
ninja-build \
pkg-config \
bison \
libfl-dev \
libbenchmark-dev \
libgmock-dev \
libz-dev
- name: Install cached non packaged dependencies
working-directory: deps
run: ../docker/build_deps.sh
- name: Build BlazingMQ
env:
PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
run: |
cmake -S . -B build/blazingmq -G Ninja \
-DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
-DCMAKE_BUILD_TYPE=Debug \
-DBDE_BUILD_TARGET_SAFE=ON \
-DBDE_BUILD_TARGET_64=ON \
-DBDE_BUILD_TARGET_CPP17=ON \
-DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
-DCMAKE_INSTALL_LIBDIR=lib64
cmake --build build/blazingmq --parallel 8 --target all
- name: Clean-up build directories before caching
run: |
find . -name "*.o" -type f -delete
find . -name "*.a" -type f -delete
- uses: actions/cache@v4
with:
path: |
build/blazingmq
deps
/opt/bb/include
key: cache-${{ github.sha }}
unit_tests_cxx:
name: UT [c++]
runs-on: ubuntu-latest
needs: build_dependencies
steps:
- uses: actions/checkout@v4
- name: Get dependencies hash
id: get-hash
run: echo "deps_hash=`cat docker/build_deps.sh | shasum`" >> $GITHUB_OUTPUT
- uses: actions/cache/restore@v4
with:
path: deps
key: deps-${{ steps.get-hash.outputs.deps_hash }}
- name: Set up dependencies
run: |
sudo apt-get update
sudo apt-get install -qy build-essential \
gdb \
curl \
python3.10 \
python3-pip \
cmake \
ninja-build \
pkg-config \
bison \
libfl-dev \
libbenchmark-dev \
libgmock-dev \
libz-dev
- name: Install cached non packaged dependencies
working-directory: deps
run: ../docker/build_deps.sh
- name: Build BlazingMQ UTs
env:
PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
run: |
cmake -S . -B build/blazingmq -G Ninja \
-DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
-DCMAKE_BUILD_TYPE=Debug \
-DBDE_BUILD_TARGET_SAFE=ON \
-DBDE_BUILD_TARGET_64=ON \
-DBDE_BUILD_TARGET_CPP17=ON \
-DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
-DCMAKE_INSTALL_LIBDIR=lib64
cmake --build build/blazingmq --parallel 8 --target all.t
- name: Run C++ Unit Tests
run: |
cd ${{ github.workspace }}/build/blazingmq
ctest -E bmqsys_executil.t --output-on-failure
unit_tests_python:
name: UT [python]
runs-on: ubuntu-latest
needs: build_ubuntu
steps:
- uses: actions/checkout@v4
- uses: actions/cache/@v4
with:
path: |
build/blazingmq
deps
/opt/bb/include
key: cache-${{ github.sha }}
- name: Run Python Unit Tests
env:
PYTHONPATH: ${{ github.workspace }}/src/python
run: |
pip install -r ${{ github.workspace }}/src/python/requirements.txt
pip install "xsdata[cli]"
cd ${{ github.workspace }}
src/python/bin/schemagen
src/python/bin/tweakgen
pytest src/python
integration_tests_ubuntu:
name: IT [${{ matrix.cluster }}/${{ matrix.mode }}]
strategy:
matrix:
mode: ["legacy_mode", "fsm_mode"]
cluster: ["single", "multi"]
fail-fast: false
runs-on: ubuntu-latest
needs: build_ubuntu
steps:
- uses: actions/checkout@v4
- uses: actions/cache/@v4
with:
path: |
build/blazingmq
deps
/opt/bb/include
key: cache-${{ github.sha }}
- name: Run Integration Tests
run: |
pip install -r ${{ github.workspace }}/src/python/requirements.txt
${{ github.workspace }}/src/integration-tests/run-tests "${{ matrix.mode }} and ${{ matrix.cluster }}" \
--log-level ERROR \
--log-file-level=info \
--bmq-tolerate-dirty-shutdown \
--bmq-log-dir=failure-logs \
--bmq-log-level=INFO \
--junitxml=integration-tests.xml \
--tb long \
--reruns=3 \
-n 4 -v
fuzz_tests_ubuntu:
name: Fuzz test [${{ matrix.request }}]
strategy:
matrix:
request: ["identity", "open_queue", "configure_queue_stream", "put", "confirm", "close_queue", "disconnect"]
fail-fast: false
runs-on: ubuntu-latest
needs: build_ubuntu
steps:
- uses: actions/checkout@v4
- uses: actions/cache/@v4
with:
path: |
build/blazingmq
deps
/opt/bb/include
key: cache-${{ github.sha }}
- name: Run Fuzz Test
run: |
pip install -r ${{ github.workspace }}/src/python/requirements.txt
cd src/python
python3 -m blazingmq.dev.fuzztest --broker-dir ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr --request ${{ matrix.request }}
build_macosx:
name: Build [macosx_${{ matrix.arch }}]
runs-on: ${{ matrix.os }}
strategy:
matrix:
include:
- os: macos-14
arch: arm64
steps:
- uses: actions/checkout@v4
- name: Set up dependencies
run: |
brew install \
curl \
[email protected] \
ninja \
bison \
flex \
google-benchmark
- name: Build BlazingMQ
run: bin/build-darwin.sh
build_and_test_bmqprometheus_plugin:
name: "Build Prometheus plugin [ubuntu]"
runs-on: ubuntu-latest
needs: build_ubuntu
steps:
- uses: actions/checkout@v4
- uses: actions/cache/@v4
with:
path: |
build/blazingmq
deps
/opt/bb/include
key: cache-${{ github.sha }}
- name: Set up plugins dependencies
run: |
sudo apt-get update
sudo apt-get install -qy build-essential \
gdb \
curl \
python3.10 \
cmake \
ninja-build \
pkg-config \
bison \
libfl-dev \
libbenchmark-dev \
libgmock-dev \
libz-dev \
autoconf \
libtool
- name: Create dependency fetcher working directory
run: mkdir -p deps
- name: Fetch & Build non packaged plugins dependencies
working-directory: deps
run: ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh
- name: Build plugins
env:
PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
run: |
cmake -S . -B build/blazingmq -G Ninja \
-DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem/toolchains/linux/gcc-default.cmake \
-DCMAKE_BUILD_TYPE=Debug \
-DBDE_BUILD_TARGET_SAFE=ON \
-DBDE_BUILD_TARGET_64=ON \
-DBDE_BUILD_TARGET_CPP17=ON \
-DCMAKE_PREFIX_PATH=${{ github.workspace }}/deps/srcs/bde-tools/BdeBuildSystem \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DINSTALL_TARGETS=prometheus
cmake --build build/blazingmq --parallel 8 --target all
- name: Create prometheus dir
run: mkdir -p prometheus_dir
- name: Download Prometheus
run: curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/
- name: Run Prometheus
run: ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
- name: Run Pushgateway
run: |
docker pull prom/pushgateway
docker run -d -p 9091:9091 prom/pushgateway
- name: Run BMQPrometheus plugin integration test in "pull" mode
run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker
- name: Clear Prometheus database and restart
run: |
curl -X POST "http://localhost:9090/-/quit"
rm -rf prometheus_dir/data
./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/src/plugins/bmqprometheus/tests/prometheus_localhost.yaml --web.enable-lifecycle --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/data &
- name: Run Prometheus plugin integration test in "push" mode
run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker
documentation:
name: "Documentation"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up dependencies
run: |
sudo apt-get update
sudo apt-get install -qy doxygen
- name: Build docs
run: |
doxygen Doxyfile
shellcheck:
name: Shellcheck
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Run ShellCheck
uses: ludeeus/action-shellcheck@master
docker_build_ubuntu:
name: "Docker [ubuntu]"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Docker Single-Node Workflow
run: docker compose -f docker/single-node/docker-compose.yaml up --build -d
- name: Docker Cluster Workflow
run: docker compose -f docker/cluster/docker-compose.yaml up --build -d