Add building wheels using cibuildwheel (#1155)

* use cibuildwheel for creating redistributable wheels * simplify CI pipeline * update packaging docs --------- Co-authored-by: Nicolas Cornu <[email protected]> Co-authored-by: Luc Grosheintz <[email protected]>
BlueBrain · May 21, 2024 · 7eb8c38 · 7eb8c38
1 parent 43ca979
commit 7eb8c38
Show file tree

Hide file tree

Showing 6 changed files with 107 additions and 227 deletions.
diff --git a/INSTALL.rst b/INSTALL.rst
@@ -180,6 +180,8 @@ libpywrapper two environment variables need to be present:
 building without linking against libpython we must set ``NMODL_PYLIB``
 before running cmake!
 
+.. _testing-installed-module:
+
 Testing the Installed Module
 ----------------------------
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -194,25 +194,13 @@ stages:
       env:
         SHELL: 'bash'
       displayName: 'Build Neuron and Run Integration Tests'
+
   - job: 'manylinux_wheels'
     timeoutInMinutes: 45
     pool:
       vmImage: 'ubuntu-20.04'
-    strategy:
-      matrix:
-        ${{ if eq(variables.buildWheel, True) }}:
-          Python38:
-            python.version: '3.8'
-          Python39:
-            python.version: '3.9'
-          Python310:
-            python.version: '3.10'
-          Python311:
-            python.version: '3.11'
-        ${{ if eq(variables.buildWheel, False) }}:
-          Python311:
-            python.version: '3.11'
     steps:
+    - task: UsePythonVersion@0
     - checkout: self
       submodules: True
       condition: succeeded()
@@ -222,99 +210,85 @@ stages:
         else
           export TAG=""
         fi
-        # the following 2 lines are a workaround for PEP 621 not allowing a
-        # dynamic `name` in `pyproject.toml`
-        python3 -m pip install --user tomli tomli-w
+        python3 -m pip install --upgrade pip
+        python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w
+        # change the name accordingly
         python3 packaging/change_name.py pyproject.toml "NMODL${TAG}"
         export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)"
-        docker run --rm \
-          -w /root/nmodl \
-          -v $PWD:/root/nmodl \
-          -e NMODL_NIGHTLY_TAG=$TAG \
-          -e SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION \
-          'bluebrain/nmodl:wheel' \
-          packaging/build_wheels.bash linux $(python.version)
+        CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python3 -m cibuildwheel --output-dir wheelhouse
       condition: succeeded()
-      displayName: 'Building ManyLinux Wheel'
+      displayName: 'Building ManyLinux Wheels'
     - task: PublishBuildArtifacts@1
       inputs:
         pathToPublish: '$(Build.SourcesDirectory)/wheelhouse'
       condition: succeeded()
       displayName: 'Publish wheel as build artifact'
-    - script: |
-        sudo apt-add-repository -y ppa:deadsnakes/ppa
-        sudo apt-get update
-        sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv
-        packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl
-      condition: succeeded()
-      displayName: 'Test ManyLinux Wheel with Python $(python.version)'
     - template: ci/upload-wheels.yml
-  - job: 'macos_wheels'
+
+  - job: 'macos_wheels_x86_64'
     timeoutInMinutes: 45
     pool:
-      vmImage: 'macOS-11'
-    strategy:
-      matrix:
-        ${{ if eq(variables.buildWheel, True) }}:
-          Python38:
-            python.version: '3.8'
-            python.org.version: '3.8.10'
-            python.installer.name: 'macosx10.9.pkg'
-          Python39:
-            python.version: '3.9'
-            python.org.version: '3.9.13'
-            python.installer.name: 'macosx10.9.pkg'
-          Python310:
-            python.version: '3.10'
-            python.org.version: '3.10.5'
-            python.installer.name: 'macos11.pkg'
-          Python311:
-            python.version: '3.11'
-            python.org.version: '3.11.1'
-            python.installer.name: 'macos11.pkg'
-        ${{ if eq(variables.buildWheel, False) }}:
-          Python311:
-            python.version: '3.11'
-            python.org.version: '3.11.1'
-            python.installer.name: 'macos11.pkg'
+      vmImage: 'macOS-12'
     steps:
     - checkout: self
       submodules: True
       condition: succeeded()
     - script: |
         brew install flex bison cmake ninja
-        export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH;
       condition: succeeded()
       displayName: 'Install Dependencies'
+    - task: UsePythonVersion@0
     - script: |
-        installer=python-$(python.org.version)-$(python.installer.name)
-        url=https://www.python.org/ftp/python/$(python.org.version)/$installer
-        curl $url -o $installer
-        sudo installer -pkg $installer -target /
-      condition: succeeded()
-      displayName: 'Install Python from python.org'
-    - script: |
-        export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH
-        export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
         if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then
           export NMODL_NIGHTLY_TAG="-nightly"
         else
           export NMODL_NIGHTLY_TAG=""
         fi
-        # the following 2 lines are a workaround for PEP 621 not allowing a
-        # dynamic `name` in `pyproject.toml`
-        python3 -m pip install tomli tomli-w
+        python3 -m pip install --upgrade pip
+        python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w
+        # change the name accordingly
         python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}"
-        SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" packaging/build_wheels.bash osx $(python.version)
+        SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse
       condition: succeeded()
-      displayName: 'Build macos Wheel'
+      displayName: 'Build macos Wheel (x86_64)'
     - task: PublishBuildArtifacts@1
       inputs:
         pathToPublish: '$(Build.SourcesDirectory)/wheelhouse'
       condition: succeeded()
       displayName: 'Publish wheel as build artifact'
+    - template: ci/upload-wheels.yml
+
+  - job: 'test_manylinux_wheels'
+    dependsOn: 'manylinux_wheels'
+    timeoutInMinutes: 45
+    pool:
+      vmImage: 'ubuntu-20.04'
+    strategy:
+      matrix:
+        ${{ if eq(variables.buildWheel, True) }}:
+          Python38:
+            python.version: '3.8'
+          Python39:
+            python.version: '3.9'
+          Python310:
+            python.version: '3.10'
+          Python311:
+            python.version: '3.11'
+          Python312:
+            python.version: '3.12'
+        ${{ if eq(variables.buildWheel, False) }}:
+          Python311:
+            python.version: '3.11'
+    steps:
+    - download: current
+      patterns: 'drop/*.whl'
+      displayName: "Make manylinux wheels available"
     - script: |
-        packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl
+        set -eux
+        sudo apt-add-repository -y ppa:deadsnakes/ppa
+        sudo apt-get update
+        sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv
+        dotless_version="$(echo "$(python.version)" | tr -d '.')"
+        find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \;
       condition: succeeded()
-      displayName: 'Test macos Wheel with Python $(python.version)'
-    - template: ci/upload-wheels.yml
+      displayName: 'Test manylinux Wheel with Python $(python.version)'
diff --git a/ci/upload-wheels.yml b/ci/upload-wheels.yml
@@ -1,7 +1,5 @@
 steps:
   - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '$(python.version)'
   - task: TwineAuthenticate@1
     inputs:
       pythonUploadServiceConnection: AzureNMODLPypiNightly

diff --git a/packaging/README.rst b/packaging/README.rst
@@ -4,8 +4,8 @@ Building Python Wheels
 Note: This is only slightly adapted from NEURONs
 `scripts <https://github.com/neuronsimulator/nrn/tree/master/packaging/python>`__.
 
-NMODL wheels are built in a manylinux2010 image. Since the generic
-docker image is very basic (CentOS 6) a new image, which brings updated
+NMODL wheels are built in a manylinux2014 image. Since the generic
+docker image is very basic (CentOS 7), a new image, which brings updated
 cmake3 (3.12), flex and bison was prepared and made available at
 https://hub.docker.com/r/bluebrain/nmodl (tag: wheel).
 
@@ -18,64 +18,66 @@ Linux `here <https://docs.docker.com/engine/install/ubuntu/>`__ and on
 OS X `here <https://docs.docker.com/docker-for-mac/install/>`__. On
 Ubuntu system we typically do:
 
-::
+.. code::sh
 
    sudo apt install docker.io
    sudo groupadd docker
    sudo usermod -aG docker $USER
 
 Logout and log back in to have docker service properly configured.
 
-Pull and start the docker image
--------------------------------
+Launch the wheel building
+-------------------------
 
-We mount local neuron repository inside docker as a volume to preserve
-any code changed. We can use -v option to mount the local folder as:
+For building the wheel we use the ``cibuilwwheel`` utility, which can be installed using:
 
-::
+.. code::sh
 
-   docker run -v /home/user/nmodl:/root/nmodl -it bluebrain/nmodl:wheel bash
+   pip install cibuildwheel
 
-where ``/home/user/nmodl`` is the NMODL repository on the host machine.
-We mount this directory inside docker at location ``/root/nmodl`` inside
-the container.
+Then to build a wheel for the current platform, run:
 
-Note that for OS X there is no docker image but on a system where all
-dependencies exist, you have to perform next building step.
+.. code::sh
 
-Launch the wheel building
--------------------------
+   cibuildwheel
 
-Once we are inside docker container, we can start building wheels. There
-is a build script that loops over the pythons ``>=3.5`` in
-``/opt/python``, build and audit the generated wheels. Results are
-placed in this wheelhouse directory.
+If you have Docker installed, you can also build the Linux wheels using:
 
-::
+.. code::sh
 
-   cd /root/nmodl
-   bash packaging/python/build_wheels.bash linux
+   cibuildwheel --platform linux
 
-For OSX on a system with the all dependencies you have to clone the
-NMODL repository and do:
+Note that, if you happen to have Podman installed instead of Docker, you can
+set ``CIBW_CONTAINER_ENGINE=podman`` to use Podman instead of Docker for this
+task.
 
-::
+Furthermore, in order to build wheels on MacOS, you need to install an official
+CPython installer from `python.org <https://www.python.org>`__.
 
-   cd nrn
-   bash packaging/python/build_wheels.bash osx
+For a complete list of all available customization options for
+``cibuildwheel``, please consult the
+`documentation <https://cibuildwheel.readthedocs.io/en/stable/options/>`__.
 
-Updating neuron_wheel docker image
-----------------------------------
+Testing the wheel
+-----------------
+
+On MacOS, the testing of the wheel is launched automatically when running
+``cibuildwheel``. On Linux, you will need to test the wheel manually, please
+see :ref:`testing-installed-module` for the instructions.
+
+
+Updating the NMODL Docker image
+-------------------------------
 
-If you have changed Dockerfile, you can build the new image as:
+If you have changed the Dockerfile, you can build the new image as:
 
-::
+.. code::sh
 
    docker build -t bluebrain/nmodl:tag .
 
-and then push image to hub.docker.com as:
+and then push the image to hub.docker.com as:
 
-::
+.. code::sh
 
    docker login --username=<username>
-   docker push bluebrain/nmodl:tag 
+   docker push bluebrain/nmodl:tag