Use cibuildwheel for all architectures; add support for aarch64 l…

…inux; publish `musllinux` wheels as well as `manylinux`; move `cibuildwheel` configuration to `pyproject.toml`; use pypa action for PyPI upload (#118) * test, deactivated workflow * test, copied cibuildwheel from website * test * test * test * test * pytest * test * linux install blas * test * test * test * test * typo * test * test * test, search on linux archs * test, search on linux archs * test, search on linux archs * test, search on linux archs * was missing qemu setup in GH runner * wrong override param from the yaml * aarch64 * maybe was missing lapack * test using miniconda, can always revert * wget curl * wget * retry * retry, conda more problems * retry musllinux * trying windows build * trying windows build * trying windows build * trying windows build * trying windows build * trying windows build * trying windows build, openblas via conda * trying windows build, openblas and python via conda * typos * typos * windows * windows * windows * windows * re-enabled all that work so far * disabled some musllinuxes * disabled some musllinuxes * testing pypi * typo * syntax yaml errors * Update cibuildwheel.yml * syntax yaml errors * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * aarch64 linux works! now cleaning and then merging * figuring which musllinux work * cleaning up, about to reopen MR * renaming * typo * typo * renamed to original file, merged * disabled win py3.8 mkl test * added macros to scspy C code * broke something with PyTypeObject * trying 3.7 * trying 3.8 * trying 3.9 * from docs it appears pytypeobject was added to ABI3 in 3.10 * trying 3.11 * Revert "trying 3.11" This reverts commit f4c33e2. * Revert "from docs it appears pytypeobject was added to ABI3 in 3.10" This reverts commit aa94da8. * Revert "trying 3.9" This reverts commit a19593d. * Revert "trying 3.8" This reverts commit 27fb7b1. * Revert "trying 3.7" This reverts commit c1ebc02. * Revert "broke something with PyTypeObject" This reverts commit 44daaed. * Revert "added macros to scspy C code" This reverts commit 29aafd2. * cleaned pyproj; put back logic to upload on push new tag * further cleaning build.yaml; no need for secrets envs --------- Co-authored-by: Brendan O'Donoghue <[email protected]>
bodono · Nov 25, 2024 · 55647d9 · 55647d9
1 parent fa64e28
commit 55647d9
Show file tree

Hide file tree

Showing 2 changed files with 121 additions and 92 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,13 +9,6 @@ on:
       - '*'
 
 jobs:
-  cleanup-runs:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: rokroskar/workflow-run-cleanup-action@master
-      env:
-        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-    if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'"
 
   build_openmp:
     runs-on: ${{ matrix.os }}
@@ -50,7 +43,7 @@ jobs:
           pytest
           rm -rf build/
 
-  build:
+  build_mkl:
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -62,7 +55,7 @@ jobs:
         # runners have Apple silicon chips.
         os: [ ubuntu-latest, macos-13, windows-latest ]
         python-version: [ 3.9, "3.10", "3.11", "3.12", "3.13"]
-        link_mkl: [true, false]
+        link_mkl: [true]
 
     env:
       PYTHON_VERSION: ${{ matrix.python-version }}
@@ -76,7 +69,6 @@ jobs:
       - name: Set Additional Envs
         shell: bash
         run: |
-          echo "DEPLOY=$( [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/tags'* ]] && echo 'True' || echo 'False' )" >> $GITHUB_ENV
           echo "PYTHON_SUBVERSION=$(echo $PYTHON_VERSION | cut -c 3-)" >> $GITHUB_ENV
       - uses: conda-incubator/setup-miniconda@v3
         with:
@@ -112,110 +104,86 @@ jobs:
         run: |
           pytest
           rm -rf build/
-      - name: Build and test windows wheels
-        if: ${{startsWith(matrix.os, 'windows') && !matrix.link_mkl}}
-        run: |
-          python -m pip install build
-          python -m build -Csetup-args="-Dlink_blas_statically=True"
-          python -m pip install delvewheel
-          delvewheel repair dist/*whl
-          conda remove openblas # to check static linkage
-          pip install wheelhouse/*whl --force-reinstall
-          pytest
-      - name: Upload artifacts to github
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.os }}-3-${{ env.PYTHON_SUBVERSION }}
-          path: ./wheelhouse
 
+  # from here to end it's a copy-paste, with few changes, of
+  # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml
 
   build_wheels:
-    needs: build
-
+    name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
       matrix:
-        os: [ ubuntu-latest, macos-13, macos-latest ]
-        python-version: [ 3.9, "3.10", "3.11", "3.12", "3.13"]
-
-    env:
-      RUNNER_OS: ${{ matrix.os }}
-      PYTHON_VERSION: ${{ matrix.python-version }}
+        # macos-13 is an intel runner, macos-14 is apple silicon
+        os: [ubuntu-latest, macos-14, windows-latest, macos-13]
 
     steps:
-
       - uses: actions/checkout@v4
         with:
-          submodules: recursive
-      - uses: actions/setup-python@v5
+          submodules: true
+
+      - name: Set up QEMU for aarch64 compilation on Linux
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Set Additional Envs
-        shell: bash
-        run: |
-          echo "PYTHON_SUBVERSION=$(echo $PYTHON_VERSION | cut -c 3-)" >> $GITHUB_ENV
-          echo "DEPLOY=$( [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/tags'* ]] && echo 'True' || echo 'False' )" >> $GITHUB_ENV
+          platforms: all
+
+      - name: Install conda on Windows
+        if: runner.os == 'Windows'
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          channels: conda-forge, anaconda
+
+      - name: Install openblas from conda on Windows
+        if: runner.os == 'Windows'
+        run: conda install -y openblas pkgconfig
 
       - name: Build wheels
-        env:
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-          CIBW_BUILD: "cp3${{env.PYTHON_SUBVERSION}}-*"
-          CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*"
-          CIBW_BEFORE_ALL_LINUX: yum install -y openblas-devel
-          CIBW_ENVIRONMENT_MACOS: CFLAGS='-Wno-error=implicit-function-declaration'
-          CIBW_BUILD_VERBOSITY: 3
-          CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}
-        uses: joerick/[email protected]
-
-      - name: Upload artifacts to github
-        uses: actions/upload-artifact@v4
+        uses: pypa/[email protected]
+
+      - uses: actions/upload-artifact@v4
         with:
-          name: wheels-${{ matrix.os }}-3-${{ env.PYTHON_SUBVERSION }}
-          path: ./wheelhouse
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
 
-  merge_wheels:
-    name: Merge wheel artifacts
+  build_sdist:
+    name: Build source distribution
     runs-on: ubuntu-latest
-    needs: build_wheels
     steps:
-      - name: Merge wheels
-        uses: actions/upload-artifact/merge@v4
+      - uses: actions/checkout@v4
         with:
-          name: wheels
-          pattern: wheels-*
-          delete-merged: true
+          submodules: true
 
-  upload_wheels:
-    needs: merge_wheels
-    runs-on: ubuntu-latest
+      - name: Build sdist
+        run: pipx run build --sdist
 
-    env:
-      PYPI_SERVER: ${{ secrets.PYPI_SERVER }}
-      PYPI_USER: ${{ secrets.PYPI_USER }}
-      PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
 
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    # We can also upload always, with skip-existing: true, below
+    # We upload on every push event (only master, above) that is a new tag
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    # Only run this step on GH release
+    # if: github.event_name == 'release' && github.event.action == 'published'
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-      - name: Set Additional Envs
-        shell: bash
-        run: |
-          echo "DEPLOY=$( [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/tags'* ]] && echo 'True' || echo 'False' )" >> $GITHUB_ENV
       - uses: actions/download-artifact@v4
-        if: ${{env.DEPLOY == 'True'}}
         with:
-          name: wheels
-          path: ./wheelhouse
+          # unpacks all CIBW artifacts into dist/
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
 
-      - name: Release to pypi
-        if: ${{env.DEPLOY == 'True'}}
-        shell: bash
-        run: |
-          python -m pip install --upgrade twine
-          twine check wheelhouse/*
-          twine upload --skip-existing --repository-url $PYPI_SERVER wheelhouse/* -u $PYPI_USER -p $PYPI_PASSWORD
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true
+        # To test:
+        # with:
+        #  repository-url: https://test.pypi.org/legacy/
diff --git a/pyproject.toml b/pyproject.toml
@@ -19,3 +19,64 @@ dependencies = [
    'numpy',
    'scipy',
 ]
+
+[tool.cibuildwheel]
+# Load SCS source submodule, already done in CI/CD
+before-all = "git submodule update --init"
+skip = [
+  "pp*", # Disable building PyPy wheels on all platforms
+  "*-win32", # fails on locating Python headers, probably meson.build is misconfigured
+  "cp37*musllinux*", # doesn't install, Scipy seems to be broken
+  "cp38*musllinux*", # doesn't install, Scipy seems to be broken
+  "*musllinux*aarch64*" # no Scipy wheels for this
+]
+# Test
+test-requires = "pytest"
+test-command = [
+  "echo wheel installed successfully",
+  "pytest {package}/test",
+]
+# we use the newer image for aarch64, for this there are openblas dev libraries
+manylinux-aarch64-image = "manylinux_2_28"
+
+[tool.cibuildwheel.linux]
+archs = [
+  "x86_64",
+  "aarch64",
+  # "i686", # various issues, may work with some more testing
+  # "s390x", # haven't tried it
+  # "ppc64le", # haven't tried it
+  # "armv7l", # haven't tried it
+  ]
+
+[tool.cibuildwheel.windows]
+# This will probably become default in newer cibuildwheels versions
+repair-wheel-command = [
+  "pip install delvewheel",
+  "delvewheel repair -w {dest_dir} {wheel}"
+]
+
+# Openblas installation for 3 different linux images
+
+[[tool.cibuildwheel.overrides]]
+select = "*-manylinux_x86_64"
+inherit.before-all = "append"
+before-all = [
+  # "yum check-update", "yum search blas",
+  # netlib blas/lapack fallback compiles and tests (on aarch64) but is super slow
+  # "((yum install -y openblas-devel) || (yum install -y blas-devel lapack-devel))",
+  "yum install -y openblas-devel"]
+
+[[tool.cibuildwheel.overrides]]
+select = "*-manylinux_aarch64"
+inherit.before-all = "append"
+before-all = [
+  # "dnf update", "dnf search blas",
+  "dnf install -y openblas-devel"]
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+inherit.before-all = "append"
+before-all = [
+  # "apk update", "apk search -v '*blas*'",
+  "apk add openblas-dev"]