From 55647d96e413f0d7f3fb19f04b9adb6fa0750d3b Mon Sep 17 00:00:00 2001 From: enzbus Date: Mon, 25 Nov 2024 14:10:52 +0400 Subject: [PATCH] Use `cibuildwheel` for all architectures; add support for `aarch64` linux; publish `musllinux` wheels as well as `manylinux`; move `cibuildwheel` configuration to `pyproject.toml`; use pypa action for PyPI upload (#118) * test, deactivated workflow * test, copied cibuildwheel from website * test * test * test * test * pytest * test * linux install blas * test * test * test * test * typo * test * test * test, search on linux archs * test, search on linux archs * test, search on linux archs * test, search on linux archs * was missing qemu setup in GH runner * wrong override param from the yaml * aarch64 * maybe was missing lapack * test using miniconda, can always revert * wget curl * wget * retry * retry, conda more problems * retry musllinux * trying windows build * trying windows build * trying windows build * trying windows build * trying windows build * trying windows build * trying windows build, openblas via conda * trying windows build, openblas and python via conda * typos * typos * windows * windows * windows * windows * re-enabled all that work so far * disabled some musllinuxes * disabled some musllinuxes * testing pypi * typo * syntax yaml errors * Update cibuildwheel.yml * syntax yaml errors * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * get aarch64 linux to work * aarch64 linux works! now cleaning and then merging * figuring which musllinux work * cleaning up, about to reopen MR * renaming * typo * typo * renamed to original file, merged * disabled win py3.8 mkl test * added macros to scspy C code * broke something with PyTypeObject * trying 3.7 * trying 3.8 * trying 3.9 * from docs it appears pytypeobject was added to ABI3 in 3.10 * trying 3.11 * Revert "trying 3.11" This reverts commit f4c33e2b3c5540b0ebf79fcfabf34d49e953f747. * Revert "from docs it appears pytypeobject was added to ABI3 in 3.10" This reverts commit aa94da819c84e9ca0e56d63ca0fa6da154dafe49. * Revert "trying 3.9" This reverts commit a19593d7b5de2830ff6e5f5b8fac5bc54abf4aa3. * Revert "trying 3.8" This reverts commit 27fb7b1e1b135eca09b8763a6581e389368a7356. * Revert "trying 3.7" This reverts commit c1ebc02730e1e94067848b7ae17d81df83c8ee04. * Revert "broke something with PyTypeObject" This reverts commit 44daaedefcf31a1506f4374303feb1cbd77b430b. * Revert "added macros to scspy C code" This reverts commit 29aafd226d92a1ac19616c56f8e3005db8ec7f78. * cleaned pyproj; put back logic to upload on push new tag * further cleaning build.yaml; no need for secrets envs --------- Co-authored-by: Brendan O'Donoghue --- .github/workflows/build.yml | 152 ++++++++++++++---------------------- pyproject.toml | 61 +++++++++++++++ 2 files changed, 121 insertions(+), 92 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de9f02c..66e3bb1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,13 +9,6 @@ on: - '*' jobs: - cleanup-runs: - runs-on: ubuntu-latest - steps: - - uses: rokroskar/workflow-run-cleanup-action@master - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master'" build_openmp: runs-on: ${{ matrix.os }} @@ -50,7 +43,7 @@ jobs: pytest rm -rf build/ - build: + build_mkl: runs-on: ${{ matrix.os }} defaults: run: @@ -62,7 +55,7 @@ jobs: # runners have Apple silicon chips. os: [ ubuntu-latest, macos-13, windows-latest ] python-version: [ 3.9, "3.10", "3.11", "3.12", "3.13"] - link_mkl: [true, false] + link_mkl: [true] env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -76,7 +69,6 @@ jobs: - name: Set Additional Envs shell: bash run: | - echo "DEPLOY=$( [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/tags'* ]] && echo 'True' || echo 'False' )" >> $GITHUB_ENV echo "PYTHON_SUBVERSION=$(echo $PYTHON_VERSION | cut -c 3-)" >> $GITHUB_ENV - uses: conda-incubator/setup-miniconda@v3 with: @@ -112,110 +104,86 @@ jobs: run: | pytest rm -rf build/ - - name: Build and test windows wheels - if: ${{startsWith(matrix.os, 'windows') && !matrix.link_mkl}} - run: | - python -m pip install build - python -m build -Csetup-args="-Dlink_blas_statically=True" - python -m pip install delvewheel - delvewheel repair dist/*whl - conda remove openblas # to check static linkage - pip install wheelhouse/*whl --force-reinstall - pytest - - name: Upload artifacts to github - if: ${{ startsWith(matrix.os, 'windows') }} - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.os }}-3-${{ env.PYTHON_SUBVERSION }} - path: ./wheelhouse + # from here to end it's a copy-paste, with few changes, of + # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml build_wheels: - needs: build - + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: - os: [ ubuntu-latest, macos-13, macos-latest ] - python-version: [ 3.9, "3.10", "3.11", "3.12", "3.13"] - - env: - RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSION: ${{ matrix.python-version }} + # macos-13 is an intel runner, macos-14 is apple silicon + os: [ubuntu-latest, macos-14, windows-latest, macos-13] steps: - - uses: actions/checkout@v4 with: - submodules: recursive - - uses: actions/setup-python@v5 + submodules: true + + - name: Set up QEMU for aarch64 compilation on Linux + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 with: - python-version: ${{ matrix.python-version }} - - name: Set Additional Envs - shell: bash - run: | - echo "PYTHON_SUBVERSION=$(echo $PYTHON_VERSION | cut -c 3-)" >> $GITHUB_ENV - echo "DEPLOY=$( [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/tags'* ]] && echo 'True' || echo 'False' )" >> $GITHUB_ENV + platforms: all + + - name: Install conda on Windows + if: runner.os == 'Windows' + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: "latest" + channels: conda-forge, anaconda + + - name: Install openblas from conda on Windows + if: runner.os == 'Windows' + run: conda install -y openblas pkgconfig - name: Build wheels - env: - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - CIBW_BUILD: "cp3${{env.PYTHON_SUBVERSION}}-*" - CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*" - CIBW_BEFORE_ALL_LINUX: yum install -y openblas-devel - CIBW_ENVIRONMENT_MACOS: CFLAGS='-Wno-error=implicit-function-declaration' - CIBW_BUILD_VERBOSITY: 3 - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: pytest {package} - uses: joerick/cibuildwheel@v2.21.3 - - - name: Upload artifacts to github - uses: actions/upload-artifact@v4 + uses: pypa/cibuildwheel@v2.21.3 + + - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-3-${{ env.PYTHON_SUBVERSION }} - path: ./wheelhouse + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl - merge_wheels: - name: Merge wheel artifacts + build_sdist: + name: Build source distribution runs-on: ubuntu-latest - needs: build_wheels steps: - - name: Merge wheels - uses: actions/upload-artifact/merge@v4 + - uses: actions/checkout@v4 with: - name: wheels - pattern: wheels-* - delete-merged: true + submodules: true - upload_wheels: - needs: merge_wheels - runs-on: ubuntu-latest + - name: Build sdist + run: pipx run build --sdist - env: - PYPI_SERVER: ${{ secrets.PYPI_SERVER }} - PYPI_USER: ${{ secrets.PYPI_USER }} - PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + - uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz + upload_pypi: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + # We can also upload always, with skip-existing: true, below + # We upload on every push event (only master, above) that is a new tag + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + # Only run this step on GH release + # if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Set Additional Envs - shell: bash - run: | - echo "DEPLOY=$( [[ $GITHUB_EVENT_NAME == 'push' && $GITHUB_REF == 'refs/tags'* ]] && echo 'True' || echo 'False' )" >> $GITHUB_ENV - uses: actions/download-artifact@v4 - if: ${{env.DEPLOY == 'True'}} with: - name: wheels - path: ./wheelhouse + # unpacks all CIBW artifacts into dist/ + pattern: cibw-* + path: dist + merge-multiple: true - - name: Release to pypi - if: ${{env.DEPLOY == 'True'}} - shell: bash - run: | - python -m pip install --upgrade twine - twine check wheelhouse/* - twine upload --skip-existing --repository-url $PYPI_SERVER wheelhouse/* -u $PYPI_USER -p $PYPI_PASSWORD + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true + # To test: + # with: + # repository-url: https://test.pypi.org/legacy/ diff --git a/pyproject.toml b/pyproject.toml index 8c17f2d..8f2947c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,64 @@ dependencies = [ 'numpy', 'scipy', ] + +[tool.cibuildwheel] +# Load SCS source submodule, already done in CI/CD +before-all = "git submodule update --init" +skip = [ + "pp*", # Disable building PyPy wheels on all platforms + "*-win32", # fails on locating Python headers, probably meson.build is misconfigured + "cp37*musllinux*", # doesn't install, Scipy seems to be broken + "cp38*musllinux*", # doesn't install, Scipy seems to be broken + "*musllinux*aarch64*" # no Scipy wheels for this +] +# Test +test-requires = "pytest" +test-command = [ + "echo wheel installed successfully", + "pytest {package}/test", +] +# we use the newer image for aarch64, for this there are openblas dev libraries +manylinux-aarch64-image = "manylinux_2_28" + +[tool.cibuildwheel.linux] +archs = [ + "x86_64", + "aarch64", + # "i686", # various issues, may work with some more testing + # "s390x", # haven't tried it + # "ppc64le", # haven't tried it + # "armv7l", # haven't tried it + ] + +[tool.cibuildwheel.windows] +# This will probably become default in newer cibuildwheels versions +repair-wheel-command = [ + "pip install delvewheel", + "delvewheel repair -w {dest_dir} {wheel}" +] + +# Openblas installation for 3 different linux images + +[[tool.cibuildwheel.overrides]] +select = "*-manylinux_x86_64" +inherit.before-all = "append" +before-all = [ + # "yum check-update", "yum search blas", + # netlib blas/lapack fallback compiles and tests (on aarch64) but is super slow + # "((yum install -y openblas-devel) || (yum install -y blas-devel lapack-devel))", + "yum install -y openblas-devel"] + +[[tool.cibuildwheel.overrides]] +select = "*-manylinux_aarch64" +inherit.before-all = "append" +before-all = [ + # "dnf update", "dnf search blas", + "dnf install -y openblas-devel"] + +[[tool.cibuildwheel.overrides]] +select = "*-musllinux*" +inherit.before-all = "append" +before-all = [ + # "apk update", "apk search -v '*blas*'", + "apk add openblas-dev"]