Refactor CI for PyPI

.cirrus.yml

@@ -3,23 +3,31 @@ env:
   DOCKERHUB_ORG: noelmni
   DOCKERHUB_REPO: antspy
   DOCKER_USERNAME: ENCRYPTED[!09cc4caa9cb6dbf5af7f2630c1d74076abd1cff6bb103db2bae45d022f28404e040ee5db21f076f70decb59a294780ab!]
-  DOCKER_PASSWORD: ENCRYPTED[!3eccd194e0044101db2d7379d2d95f9255e3493c7ae72a6843ca83a6547ea271541271d77858a2ffd30f61ab68514a64!]
-  GITHUB_TOKEN: ENCRYPTED[!PLACEHOLDER_FOR_GITHUB_TOKEN!]
-  TWINE_USERNAME: ENCRYPTED[!PLACEHOLDER_FOR_TWINE_USERNAME!] # username for pypi
-  TWINE_PASSWORD: ENCRYPTED[!PLACEHOLDER_FOR_TWINE_PASSWORD!] # password for pypi
+  DOCKER_PASSWORD: ENCRYPTED[!4e8f945da70908366e7389ee7192c8c3f13d3f0411a75442bef1af794c2ce547e83b9b60044923c9775790b622da54f1!]
+  GITHUB_TOKEN: ENCRYPTED[!67d503db87885a2486161bdc51da6492e80c522c551c8d2c9b6f871af8adb354ca825c128a7b79f6755eb8b99610732a!]
+  TWINE_USERNAME: __token__ # username for pypi
+  TWINE_PASSWORD: ENCRYPTED[!40b9d18b454ec9c49f6b2df6afda3769644014f0c4612ac36db620c27fe58964ff81138c4e28e01b8a0001291e3eba1c!] # api token for pypi
   TAG_x86_64: ci-x86_64-tmp
   TAG_AARCH64: ci-aarch64-tmp
 
 
 build_and_store_wheels: &BUILD_AND_STORE_WHEELS
   install_cibuildwheel_pipx_script:
+    - which python
+    - python -VV
+    - python -m pip install --upgrade build pip twine
     - python -m pip install cibuildwheel==2.16.2 jq pipx setuptools
     - python -m pipx ensurepath
+    - which pipx
   run_cibuildwheel_script:
-    - cibuildwheel
+    - pipx run cibuildwheel
+  wheels_artifacts:
+    path: "wheelhouse/*"
   make_sdist_script:
     - rm -rf dist # clear out your 'dist' folder 
     - pipx run build --sdist # make a source distribution
+  sdist_artifacts:
+    path: "dist/*.tar.gz"
   upload_releases_script: |
     #!/usr/bin/env bash
 
@@ -48,19 +56,15 @@ build_and_store_wheels: &BUILD_AND_STORE_WHEELS
   upload_pypi_script: |
     #!/usr/bin/env bash
 
-    if [[ "${CIRRUS_RELEASE}" == "" ]]; then
-        exit 0
-    fi
-
-    # deploy source distribution to PyPI using 'twine'
-    pipx run twine upload -r pypi dist/*
+    # if [[ "${CIRRUS_RELEASE}" == "" ]]; then
+    #     exit 0
+    # fi
 
     # deploy wheels to PyPI
-    pipx run twine upload -r pypi wheelhouse/*
-  sdist_artifacts:
-    path: "dist/*.tar.gz"
-  wheels_artifacts:
-    path: "wheelhouse/*"
+    pipx run twine upload --repository pypi wheelhouse/*
+
+    # deploy source distribution to PyPI
+    pipx run twine upload --repository pypi dist/*
 
 
 linux_aarch64_task:
@@ -78,7 +82,7 @@ linux_aarch64_task:
     CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
     CIBW_BEFORE_ALL_LINUX: >
       yum install -y gcc-c++ libpng-devel libpng &&
-      python -m pip install cmake ninja setuptools
+      python -m pip install cmake ninja
     CIBW_BEFORE_TEST: |
       python -m pip install --find-links=wheelhouse/ -r requirements.txt
     CIBW_TEST_COMMAND: bash {project}/tests/run_tests.sh
@@ -106,27 +110,25 @@ macos_arm64_task:
   name: build_macos_arm64_wheels
   # only_if: ${CIRRUS_BRANCH} == 'master' || ${CIRRUS_PR} != ''
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:15
+    image: ghcr.io/cirruslabs/macos-sonoma-xcode:15.1
   env:
     matrix:
-      - CIBW_BUILD: cp38-macosx_arm64
       - CIBW_BUILD: cp39-macosx_arm64
       - CIBW_BUILD: cp310-macosx_arm64
       - CIBW_BUILD: cp311-macosx_arm64
       - CIBW_BUILD: cp312-macosx_arm64
     CIBW_ARCHS_MACOS: arm64
     CIBW_BEFORE_ALL_MACOS: >
       python -m ensurepip --upgrade &&
-      conda install cmake ninja libpng setuptools
+      mamba install cmake ninja libpng
     CIBW_BEFORE_TEST: |
       python -m pip install --find-links=wheelhouse/ -r requirements.txt
     CIBW_TEST_COMMAND: bash {project}/tests/run_tests.sh
-    CIBW_TEST_SKIP: cp38-macosx_*:arm64
     PATH: $HOME/mambaforge/bin/:${PATH}
     CONDA_HOME: $HOME/mambaforge
   conda_script:
-    - curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh
-    - bash ~/mambaforge.sh -b -p ~/mambaforge
+    - curl -L -o ~/mambaforge.sh https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/Mambaforge-23.3.1-1-MacOSX-arm64.sh
+    - bash ~/mambaforge.sh -b -p ${CONDA_HOME}
   <<: *BUILD_AND_STORE_WHEELS
 
 

.github/workflows/ci-docker.yml

@@ -1,4 +1,4 @@
-name: ci-docker
+name: Build docker image [x86_64]
 on:
   push:
     branches:
@@ -10,32 +10,31 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Docker meta
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: antsx/antspy
-      -
-        name: Login to DockerHub
+
+      - name: Login to DockerHub
         if: ${{ github.repository == 'ANTsX/ANTsPy' }}
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      -
-        name: Set up Docker Buildx
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
-      -
-        name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64

.github/workflows/wheels-macosx_arm64.yml

@@ -0,0 +1,151 @@
+name: Build wheels [macosx_arm64]
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+  release:
+    types: [created]
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: Build wheel for cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # macOS on Apple M1 64-bit
+          - os: [self-hosted, macOS, ARM64]
+            python: '3.9'
+            cibw_python: 39
+            arch: arm64
+            platform_id: macosx_arm64
+          - os: [self-hosted, macOS, ARM64]
+            python: '3.10'
+            cibw_python: 310
+            arch: arm64
+            platform_id: macosx_arm64
+          - os: [self-hosted, macOS, ARM64]
+            python: '3.11'
+            cibw_python: 311
+            arch: arm64
+            platform_id: macosx_arm64
+          - os: [self-hosted, macOS, ARM64]
+            python: '3.12'
+            cibw_python: 312
+            arch: arm64
+            platform_id: macosx_arm64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Install Python host for cibuildwheel
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-activate-base: false
+          activate-environment: py${{ matrix.cibw_python }}
+          installer-url: https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/Mambaforge-23.3.1-1-MacOSX-arm64.sh
+          allow-softlinks: true
+          show-channel-urls: true
+          use-only-tar-bz2: true
+          channels: conda-forge
+          channel-priority: true
+          init-shell: bash
+          cache-environment: true
+          cache-environment-key: environment-${{ steps.date.outputs.date }}
+          cache-downloads-key: downloads-${{ steps.date.outputs.date }}
+          post-cleanup: 'all'
+          use-mamba: true
+          python-version: ${{ matrix.python }}
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        run: |
+          # Workaround for https://github.com/conda-incubator/setup-miniconda/issues/186
+          conda config --remove channels defaults
+          # dependencies
+          python -m ensurepip --upgrade
+          mamba install cmake ninja libpng setuptools
+
+      - name: Install cibuildwheel and other build tools
+        shell: bash -l {0}
+        run: |
+          python -m pip install --upgrade build pip twine setuptools
+          python -m pip install cibuildwheel==2.16.2 jq pipx
+          python -m pipx ensurepath
+
+      - name: Get package name and version (Linux / Mac)
+        shell: bash -l {0}
+        if: ${{ ! startsWith(matrix.os, 'windows-') }}
+        run: |
+          echo PACKAGE_NAME=$( python setup.py --name ) >> $GITHUB_ENV
+          echo PACKAGE_VERSION=$( python setup.py --version ) >> $GITHUB_ENV
+
+      - name: Build and test wheels
+        shell: bash -l {0}
+        env:
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+          CIBW_BUILD: cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
+          # Include latest Python beta
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_ARCHS_MACOS: ${{ matrix.arch }}
+          CIBW_ENVIRONMENT_MACOS: |
+            CMAKE_OSX_ARCHITECTURES=${{ matrix.arch }}
+          CIBW_BEFORE_TEST: |
+            python -m pip install --find-links=wheelhouse/ -r requirements.txt
+          CIBW_TEST_COMMAND: bash {project}/tests/run_tests.sh
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
+
+      - name: Store wheel artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.PACKAGE_NAME }}-${{ env.PACKAGE_VERSION }}-cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
+          path: ./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}/*.whl
+
+      - name: Upload release asset
+        # Previously was using actions/upload-release-asset@v1 , but this had some
+        # errors with large files
+        uses: ncipollo/[email protected]
+        if: ${{ github.event_name == 'release' }}
+        with:
+          allowUpdates: true
+          omitBodyDuringUpdate: true
+          omitNameDuringUpdate: true
+          artifacts: ./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}/*.whl
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+
+  pypi-publish:
+    name: Upload release to PyPI
+    if: github.event_name == 'release'
+    needs: [build_wheels]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    steps:
+    # retrieve your distributions here
+    - name: Download artifacts
+      uses: actions/download-artifact@v4
+      with:
+        # unpacks all CIBW artifacts into dist/
+        pattern: antspyx*
+        path: dist
+        merge-multiple: true
+
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        verify-metadata: false
+        skip-existing: true
+        verbose: true