#2 Add GitHub Actions to Python SDK

* CI: Update tests so they lint properly Signed-off-by: Patrick M. Niedzielski <[email protected]> * CI: Add DCO Check for PRs This project requires contributors to verify that they are releasing their changes under the Apache 2.0 license that we use, by signing off on a DCO. This patch adds a CI check that ensures each commit in a PR has a `Signed-off-by` trailer indicating that the contributor has agreed to our DCO. Signed-off-by: Patrick M. Niedzielski <[email protected]> * CI: Add News Check We are using `towncrier` to generate the changelog for this project, and `towncrier` reads changes out of the `news/` directory. This patch ensures that new PRs that change the functionality of the SDK either come with a file in the `news/` directory that explains their changes or is explicitly marked with a `skip news` tag. Signed-off-by: Patrick M. Niedzielski <[email protected]> * CI: Add action to build and test the library Signed-off-by: Patrick M. Niedzielski <[email protected]> * CI: Add an environment variable to turn off pkg-config checks We want to be able to generate sdists from our `setup.py` without requiring BlazingMQ to be on the system (as it is not, for instance, on a GitHub Action container). This patch introduces an environment variable for this purpose. Signed-off-by: Patrick M. Niedzielski <[email protected]> * CI: Build wheels Signed-off-by: Patrick M. Niedzielski <[email protected]> --------- Signed-off-by: Patrick M. Niedzielski <[email protected]>
bloomberg · Nov 7, 2023 · 871d55d · 871d55d
1 parent b851dba
commit 871d55d
Show file tree

Hide file tree

Showing 37 changed files with 680 additions and 48 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,229 @@
+name: Tests
+
+on:
+  push:
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  blazingmq-dependency:
+    name: Build BlazingMQ as a dependency
+    runs-on: ubuntu-latest
+    outputs:
+      blazingmq_sha: ${{ steps.get-sha.outputs.blazingmq_sha }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Checkout BlazingMQ
+        run:  git clone --depth 1 https://github.com/bloomberg/blazingmq
+
+      - name: Get latest BlazingMQ commit SHA
+        id: get-sha
+        working-directory: blazingmq
+        run: echo "blazingmq_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Try to get cached BlazingMQ build artifacts
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: blazingmq_artifacts.tar.gz
+          key: ${{ steps.get-sha.outputs.blazingmq_sha }}
+
+      - name: Set up dependencies
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: |
+            sudo apt-get update
+            sudo apt-get install -qy build-essential \
+              gdb \
+              curl \
+              python3.10 \
+              cmake \
+              ninja-build \
+              pkg-config \
+              bison \
+              libfl-dev \
+              libbenchmark-dev \
+              libz-dev
+
+      - name: Create install directory for BlazingMQ and its dependencies
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: mkdir -p blazingmq_artifacts
+
+      - name: Skip building bison and google-benchmark
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p thirdparty/{bison,google-benchmark}
+          mkdir -p build/{bison,google-benchmark}
+          touch thirdparty/bison/.complete
+          touch build/google-benchmark/.complete
+
+      - name: Build and install BlazingMQ and its dependencies
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        env:
+          DIR_INSTALL: 'blazingmq_artifacts'
+        run: /bin/bash bin/build-manylinux.sh
+
+      - name: Save built BlazingMQ build artifacts
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        run: tar czf blazingmq_artifacts.tar.gz blazingmq_artifacts
+
+      - name: Cache built BlazingMQ build artifacts
+        id: cache-save
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: blazingmq_artifacts.tar.gz
+          key: ${{ steps.get-sha.outputs.blazingmq_sha }}
+
+  linux-check:
+    name: Test on Linux
+    needs: blazingmq-dependency
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Try to get cached BlazingMQ build artifacts
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: blazingmq_artifacts.tar.gz
+          key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }}
+      - name: Restore cached BlazingMQ build artifacts
+        run: tar xzf blazingmq_artifacts.tar.gz
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Create virtual environment
+        run: |
+          python3 -m venv venv
+      - name: Install Python dependencies
+        run: |
+          ./venv/bin/python -m pip install --upgrade pip
+          ./venv/bin/python -m pip install -r requirements-dev.txt
+      - name: Install package manager dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential \
+            gdb \
+            curl \
+            cmake \
+            cmake \
+            ninja-build \
+            pkg-config \
+            bison \
+            libfl-dev \
+            libbenchmark-dev \
+            libz-dev
+      - name: Run tests
+        env:
+          BMQ_BROKER_URI: tcp://localhost:30114
+          PREFIX: blazingmq_artifacts
+          PYTHON: ./venv/bin/python
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig
+        run: |
+          mkdir -p bmq/logs
+          mkdir -p bmq/storage/archive
+          ./blazingmq_artifacts/bin/bmqbrkr.tsk ./tests/broker-config &
+          (sleep 5; make test-build && make test-install && make check)
+
+  lint-docs:
+    name: Lint and Docs
+    needs: blazingmq-dependency
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Try to get cached BlazingMQ build artifacts
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: blazingmq_artifacts.tar.gz
+          key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }}
+      - name: Restore cached BlazingMQ build artifacts
+        run: tar xzf blazingmq_artifacts.tar.gz
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Create virtual environment
+        run: |
+          python3 -m venv venv
+      - name: Install Python dependencies
+        run: |
+          ./venv/bin/python -m pip install --upgrade pip
+          ./venv/bin/python -m pip install -r requirements-dev.txt
+      - name: Set up dependencies
+        run:
+          sudo apt-get install clang-format
+      - name: Install Package
+        run: |
+          ./venv/bin/python -m pip install -e .
+        env:
+          PREFIX: blazingmq_artifacts
+          PYTHON: ./venv/bin/python
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig
+      - name: Lint sources
+        env:
+          PYTHON: ./venv/bin/python
+        run: |
+          make lint
+      - name: Build docs
+        run: |
+          ./venv/bin/python -m towncrier build --version 99.99 --name blazingmq --keep
+          make docs
+        env:
+          PYTHON: ./venv/bin/python
+          SPHINXBUILD: ../venv/bin/sphinx-build
+
+  coverage:
+    name: Coverage
+    needs: blazingmq-dependency
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Try to get cached BlazingMQ build artifacts
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: blazingmq_artifacts.tar.gz
+          key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }}
+      - name: Restore cached BlazingMQ build artifacts
+        run: tar xzf blazingmq_artifacts.tar.gz
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Create virtual environment
+        run: |
+          python3 -m venv venv
+      - name: Install Python dependencies
+        run: |
+          ./venv/bin/python3 -m pip install --upgrade pip
+          ./venv/bin/python3 -m pip install -r requirements-dev.txt
+      - name: Run tests with coverage
+        env:
+          BMQ_BROKER_URI: tcp://localhost:30114
+          PREFIX: blazingmq_artifacts
+          PYTHON: ./venv/bin/python
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig
+        run: |
+          mkdir -p bmq/logs
+          mkdir -p bmq/storage/archive
+          ./blazingmq_artifacts/bin/bmqbrkr.tsk ./tests/broker-config &
+          (sleep 5; make coverage-install && make coverage)
+      - name: Output code coverage summary
+        uses: irongut/[email protected]
+        with:
+          filename: coverage*.xml
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -2,10 +2,6 @@ name: Wheels
 
 on:
   push:
-  pull_request:
-    paths-ignore:
-      - "docs/**"
-      - "CONTRIBUTING.md"
   release:
     types:
       - published
@@ -25,6 +21,8 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Build sdist
+        env:
+          BLAZINGMQ_PYTHON_NO_PKGCONFIG: ’1’
         run: pipx run build --sdist
 
       - uses: actions/upload-artifact@v3
@@ -37,14 +35,17 @@ jobs:
           name: tests
           path: tests
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: bin
+          path: bin
+
   choose_architectures:
     name: Decide which architectures to build wheels for
     runs-on: ubuntu-latest
     steps:
       - id: x86_64
         run: echo "cibw_arch=x86_64" >> $GITHUB_OUTPUT
-      - id: i686
-        run: echo "cibw_arch=i686" >> $GITHUB_OUTPUT
       - id: aarch64
         if: github.event_name == 'release' && github.event.action == 'published'
         run: echo "cibw_arch=aarch64" >> $GITHUB_OUTPUT
@@ -72,6 +73,10 @@ jobs:
         with:
           name: tests
           path: tests
+      - uses: actions/download-artifact@v3
+        with:
+          name: bin
+          path: bin
       - uses: docker/setup-qemu-action@v2
         if: runner.os == 'Linux'
         name: Set up QEMU
@@ -84,12 +89,11 @@ jobs:
       - name: Build wheels
         uses: pypa/[email protected]
         env:
-          CIBW_BUILD: ${{ matrix.cibw_python }}
           CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }}
+          CIBW_BUILD: ${{ matrix.cibw_python }}
           CIBW_PRERELEASE_PYTHONS: True
-          CIBW_TEST_EXTRAS: test
-          CIBW_TEST_COMMAND: python -m pytest {package}/tests
-          CIBW_TEST_SKIP: "*aarch64*"
+          CIBW_TEST_COMMAND: python3 -m pytest {project}/tests/unit
+          CIBW_TEST_REQUIRES: pytest mock pkgconfig
       - uses: actions/upload-artifact@v3
         with:
           name: dist
@@ -102,8 +106,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-11]
-        cibw_python: ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"]
+        os: [macos-12]
+        cibw_python: ["cp311-*"]
         cibw_arch: ["x86_64", "arm64"]
 
     steps:
@@ -115,22 +119,26 @@ jobs:
         with:
           name: tests
           path: tests
+      - uses: actions/download-artifact@v3
+        with:
+          name: bin
+          path: bin
       - name: Extract sdist
         run: |
           tar zxvf dist/*.tar.gz --strip-components=1
       - name: Sets env vars for compilation
         if: matrix.cibw_arch == 'arm64'
         run: |
-          echo "CFLAGS=-target arm64-apple-macos11" >> $GITHUB_ENV
+          echo "CFLAGS=-target arm64-apple-macos12" >> $GITHUB_ENV
       - name: Build wheels
         uses: pypa/[email protected]
         env:
-          CIBW_BUILD: ${{ matrix.cibw_python }}
           CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }}
-          CIBW_PRERELEASE_PYTHONS: True
-          CIBW_TEST_EXTRAS: test
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_BUILD: ${{ matrix.cibw_python }}
           CIBW_BUILD_VERBOSITY: 1
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_TEST_COMMAND: python3 -m pytest {project}/tests/unit
+          CIBW_TEST_REQUIRES: pytest mock pkgconfig
           MACOSX_DEPLOYMENT_TARGET: "11.0"
 
       - uses: actions/upload-artifact@v3

diff --git a/.github/workflows/dco-check.yaml b/.github/workflows/dco-check.yaml
@@ -0,0 +1,24 @@
+name: DCO check
+on:
+  pull_request:
+    types:
+      - "opened"
+      - "reopened"
+      - "synchronize"
+      - "labeled"
+      - "unlabeled"
+
+jobs:
+  commits_check_job:
+    runs-on: ubuntu-latest
+    name: Commits Check
+    steps:
+      - name: Get PR Commits
+        id: "get-pr-commits"
+        uses: tim-actions/get-pr-commits@master
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: DCO Check
+        uses: tim-actions/dco@master
+        with:
+          commits: ${{ steps.get-pr-commits.outputs.commits }}
diff --git a/.github/workflows/news-check.yaml b/.github/workflows/news-check.yaml
@@ -0,0 +1,23 @@
+name: News entry check
+on:
+  pull_request:
+    paths:
+      - "src/**"
+    types:
+      - "opened"
+      - "reopened"
+      - "synchronize"
+      - "labeled"
+      - "unlabeled"
+
+jobs:
+  news_entry_check:
+    runs-on: ubuntu-latest
+    name: Check for news entry
+    steps:
+      - name: "Check for news entry"
+        uses: brettcannon/check-for-changed-files@v1
+        with:
+          file-pattern: "news/*.rst"
+          skip-label: "skip news"
+          failure-message: "Missing a news file in ${file-pattern}; please add one or apply the ${skip-label} label to the pull request"
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -5,4 +5,5 @@ recursive-exclude src/blazingmq *.cpp
 recursive-include src/blazingmq *.py
 recursive-include src/blazingmq *.pyx *.pxd
 recursive-include src/blazingmq *.vers
+recursive-include src/blazingmq *.exp
 recursive-include src/declarations *.pxd