From 5ad578bf33d8185cf0aa2a20d21eb7abbc8e642b Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 18 Mar 2024 01:51:26 -0400 Subject: [PATCH] gh-actions: Parallelize wheel-building by explicitly specifying wheels to build in job matrix. (#51) --- .github/workflows/build_wheels.yml | 150 ++++++++++++++++++++++++++--- pyproject.toml | 7 ++ 2 files changed, 144 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index af3ff1fc..1d2ed013 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -39,9 +39,9 @@ jobs: linters: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -62,32 +62,156 @@ jobs: build_wheels: needs: [linters] - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build ${{ matrix.build.name }} + runs-on: ${{ matrix.build.os }} strategy: matrix: - os: [ubuntu-22.04, macos-12] + build: + # macOS builds + - name: cp37-macosx_x86_64 + os: macos-12 + - name: cp38-macosx_arm64 + os: macos-12 + - name: cp38-macosx_universal2 + os: macos-12 + - name: cp38-macosx_x86_64 + os: macos-12 + - name: cp39-macosx_arm64 + os: macos-12 + - name: cp39-macosx_universal2 + os: macos-12 + - name: cp39-macosx_x86_64 + os: macos-12 + - name: cp310-macosx_arm64 + os: macos-12 + - name: cp310-macosx_universal2 + os: macos-12 + - name: cp310-macosx_x86_64 + os: macos-12 + - name: cp311-macosx_arm64 + os: macos-12 + - name: cp311-macosx_universal2 + os: macos-12 + - name: cp311-macosx_x86_64 + os: macos-12 + - name: cp312-macosx_arm64 + os: macos-12 + - name: cp312-macosx_universal2 + os: macos-12 + - name: cp312-macosx_x86_64 + os: macos-12 - steps: - - uses: actions/checkout@v3 + # Linux builds + - name: cp310-manylinux_aarch64 + os: ubuntu-22.04 + - name: cp310-manylinux_i686 + os: ubuntu-22.04 + - name: cp310-manylinux_x86_64 + os: ubuntu-22.04 + - name: cp310-musllinux_aarch64 + os: ubuntu-22.04 + - name: cp310-musllinux_i686 + os: ubuntu-22.04 + - name: cp310-musllinux_x86_64 + os: ubuntu-22.04 + - name: cp311-manylinux_aarch64 + os: ubuntu-22.04 + - name: cp311-manylinux_i686 + os: ubuntu-22.04 + - name: cp311-manylinux_x86_64 + os: ubuntu-22.04 + - name: cp311-musllinux_aarch64 + os: ubuntu-22.04 + - name: cp311-musllinux_i686 + os: ubuntu-22.04 + - name: cp311-musllinux_x86_64 + os: ubuntu-22.04 + - name: cp312-manylinux_aarch64 + os: ubuntu-22.04 + - name: cp312-manylinux_i686 + os: ubuntu-22.04 + - name: cp312-manylinux_x86_64 + os: ubuntu-22.04 + - name: cp312-musllinux_aarch64 + os: ubuntu-22.04 + - name: cp312-musllinux_i686 + os: ubuntu-22.04 + - name: cp312-musllinux_x86_64 + os: ubuntu-22.04 + - name: cp37-manylinux_aarch64 + os: ubuntu-22.04 + - name: cp37-manylinux_i686 + os: ubuntu-22.04 + - name: cp37-manylinux_x86_64 + os: ubuntu-22.04 + - name: cp37-musllinux_aarch64 + os: ubuntu-22.04 + - name: cp37-musllinux_i686 + os: ubuntu-22.04 + - name: cp37-musllinux_x86_64 + os: ubuntu-22.04 + - name: cp38-manylinux_aarch64 + os: ubuntu-22.04 + - name: cp38-manylinux_i686 + os: ubuntu-22.04 + - name: cp38-manylinux_x86_64 + os: ubuntu-22.04 + - name: cp38-musllinux_aarch64 + os: ubuntu-22.04 + - name: cp38-musllinux_i686 + os: ubuntu-22.04 + - name: cp38-musllinux_x86_64 + os: ubuntu-22.04 + - name: cp39-manylinux_aarch64 + os: ubuntu-22.04 + - name: cp39-manylinux_i686 + os: ubuntu-22.04 + - name: cp39-manylinux_x86_64 + os: ubuntu-22.04 + - name: cp39-musllinux_aarch64 + os: ubuntu-22.04 + - name: cp39-musllinux_i686 + os: ubuntu-22.04 + - name: cp39-musllinux_x86_64 + os: ubuntu-22.04 + steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.16.2 + uses: pypa/cibuildwheel@v2.17.0 env: - CIBW_ARCHS_LINUX: auto aarch64 + CIBW_BUILD: ${{ matrix.build.name }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: wheel-${{ matrix.os }} + name: wheel-${{ matrix.build.name }} path: ./wheelhouse/*.whl + retention-days: 1 + + combine-wheels: + needs: [build_wheels] + name: Combine wheels + runs-on: ubuntu-latest + env: + WHEEL_DIR: /tmp/wheels + + steps: + - uses: actions/download-artifact@v4 + with: + path: ${{env.WHEEL_DIR}} + merge-multiple: true + + - uses: actions/upload-artifact@v4 + with: + name: all-wheels + path: ${{env.WHEEL_DIR}} diff --git a/pyproject.toml b/pyproject.toml index ead8d31e..07c14d40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,10 @@ color = true preview = true [tool.cibuildwheel] +# NOTE: The wheels to be built (and skipped) must be kept in-sync with the GitHub workflow. The GH +# workflow doesn't use pyproject.toml to determine which wheels to build, but instead uses +# environment variables so that it can build the wheels in parallel jobs. + # Use `musllinux_1_2` to support C++20 compilation musllinux-aarch64-image = "musllinux_1_2" musllinux-i686-image = "musllinux_1_2" @@ -58,6 +62,9 @@ test-skip = [ "*-*linux_i686", ] +[tool.cibuildwheel.linux] +archs = ["auto", "aarch64"] + [tool.cibuildwheel.macos] archs = ["x86_64", "universal2", "arm64"]