From 7eb8c38a814d482700ce24b19676cc018f4932ed Mon Sep 17 00:00:00 2001 From: JCGoran Date: Wed, 6 Mar 2024 12:58:11 +0100 Subject: [PATCH] Add building wheels using cibuildwheel (#1155) * use cibuildwheel for creating redistributable wheels * simplify CI pipeline * update packaging docs --------- Co-authored-by: Nicolas Cornu Co-authored-by: Luc Grosheintz --- INSTALL.rst | 2 + azure-pipelines.yml | 126 ++++++++++++++---------------------- ci/upload-wheels.yml | 2 - packaging/README.rst | 72 +++++++++++---------- packaging/build_wheels.bash | 114 -------------------------------- pyproject.toml | 18 ++++++ 6 files changed, 107 insertions(+), 227 deletions(-) delete mode 100755 packaging/build_wheels.bash diff --git a/INSTALL.rst b/INSTALL.rst index 9c4fc6af46..65bf1e2ca8 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -180,6 +180,8 @@ libpywrapper two environment variables need to be present: building without linking against libpython we must set ``NMODL_PYLIB`` before running cmake! +.. _testing-installed-module: + Testing the Installed Module ---------------------------- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 663a792dcd..aba410351d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -194,25 +194,13 @@ stages: env: SHELL: 'bash' displayName: 'Build Neuron and Run Integration Tests' + - job: 'manylinux_wheels' timeoutInMinutes: 45 pool: vmImage: 'ubuntu-20.04' - strategy: - matrix: - ${{ if eq(variables.buildWheel, True) }}: - Python38: - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - Python311: - python.version: '3.11' - ${{ if eq(variables.buildWheel, False) }}: - Python311: - python.version: '3.11' steps: + - task: UsePythonVersion@0 - checkout: self submodules: True condition: succeeded() @@ -222,99 +210,85 @@ stages: else export TAG="" fi - # the following 2 lines are a workaround for PEP 621 not allowing a - # dynamic `name` in `pyproject.toml` - python3 -m pip install --user tomli tomli-w + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w + # change the name accordingly python3 packaging/change_name.py pyproject.toml "NMODL${TAG}" export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" - docker run --rm \ - -w /root/nmodl \ - -v $PWD:/root/nmodl \ - -e NMODL_NIGHTLY_TAG=$TAG \ - -e SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION \ - 'bluebrain/nmodl:wheel' \ - packaging/build_wheels.bash linux $(python.version) + CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() - displayName: 'Building ManyLinux Wheel' + displayName: 'Building ManyLinux Wheels' - task: PublishBuildArtifacts@1 inputs: pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' condition: succeeded() displayName: 'Publish wheel as build artifact' - - script: | - sudo apt-add-repository -y ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv - packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl - condition: succeeded() - displayName: 'Test ManyLinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml - - job: 'macos_wheels' + + - job: 'macos_wheels_x86_64' timeoutInMinutes: 45 pool: - vmImage: 'macOS-11' - strategy: - matrix: - ${{ if eq(variables.buildWheel, True) }}: - Python38: - python.version: '3.8' - python.org.version: '3.8.10' - python.installer.name: 'macosx10.9.pkg' - Python39: - python.version: '3.9' - python.org.version: '3.9.13' - python.installer.name: 'macosx10.9.pkg' - Python310: - python.version: '3.10' - python.org.version: '3.10.5' - python.installer.name: 'macos11.pkg' - Python311: - python.version: '3.11' - python.org.version: '3.11.1' - python.installer.name: 'macos11.pkg' - ${{ if eq(variables.buildWheel, False) }}: - Python311: - python.version: '3.11' - python.org.version: '3.11.1' - python.installer.name: 'macos11.pkg' + vmImage: 'macOS-12' steps: - checkout: self submodules: True condition: succeeded() - script: | brew install flex bison cmake ninja - export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH; condition: succeeded() displayName: 'Install Dependencies' + - task: UsePythonVersion@0 - script: | - installer=python-$(python.org.version)-$(python.installer.name) - url=https://www.python.org/ftp/python/$(python.org.version)/$installer - curl $url -o $installer - sudo installer -pkg $installer -target / - condition: succeeded() - displayName: 'Install Python from python.org' - - script: | - export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH - export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then export NMODL_NIGHTLY_TAG="-nightly" else export NMODL_NIGHTLY_TAG="" fi - # the following 2 lines are a workaround for PEP 621 not allowing a - # dynamic `name` in `pyproject.toml` - python3 -m pip install tomli tomli-w + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w + # change the name accordingly python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}" - SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" packaging/build_wheels.bash osx $(python.version) + SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() - displayName: 'Build macos Wheel' + displayName: 'Build macos Wheel (x86_64)' - task: PublishBuildArtifacts@1 inputs: pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' condition: succeeded() displayName: 'Publish wheel as build artifact' + - template: ci/upload-wheels.yml + + - job: 'test_manylinux_wheels' + dependsOn: 'manylinux_wheels' + timeoutInMinutes: 45 + pool: + vmImage: 'ubuntu-20.04' + strategy: + matrix: + ${{ if eq(variables.buildWheel, True) }}: + Python38: + python.version: '3.8' + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + Python311: + python.version: '3.11' + Python312: + python.version: '3.12' + ${{ if eq(variables.buildWheel, False) }}: + Python311: + python.version: '3.11' + steps: + - download: current + patterns: 'drop/*.whl' + displayName: "Make manylinux wheels available" - script: | - packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl + set -eux + sudo apt-add-repository -y ppa:deadsnakes/ppa + sudo apt-get update + sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv + dotless_version="$(echo "$(python.version)" | tr -d '.')" + find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() - displayName: 'Test macos Wheel with Python $(python.version)' - - template: ci/upload-wheels.yml + displayName: 'Test manylinux Wheel with Python $(python.version)' diff --git a/ci/upload-wheels.yml b/ci/upload-wheels.yml index 29e52767e2..7091ec285f 100644 --- a/ci/upload-wheels.yml +++ b/ci/upload-wheels.yml @@ -1,7 +1,5 @@ steps: - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - task: TwineAuthenticate@1 inputs: pythonUploadServiceConnection: AzureNMODLPypiNightly diff --git a/packaging/README.rst b/packaging/README.rst index eb2d8ddf7e..f4cc83cff0 100644 --- a/packaging/README.rst +++ b/packaging/README.rst @@ -4,8 +4,8 @@ Building Python Wheels Note: This is only slightly adapted from NEURONs `scripts `__. -NMODL wheels are built in a manylinux2010 image. Since the generic -docker image is very basic (CentOS 6) a new image, which brings updated +NMODL wheels are built in a manylinux2014 image. Since the generic +docker image is very basic (CentOS 7), a new image, which brings updated cmake3 (3.12), flex and bison was prepared and made available at https://hub.docker.com/r/bluebrain/nmodl (tag: wheel). @@ -18,7 +18,7 @@ Linux `here `__ and on OS X `here `__. On Ubuntu system we typically do: -:: +.. code::sh sudo apt install docker.io sudo groupadd docker @@ -26,56 +26,58 @@ Ubuntu system we typically do: Logout and log back in to have docker service properly configured. -Pull and start the docker image -------------------------------- +Launch the wheel building +------------------------- -We mount local neuron repository inside docker as a volume to preserve -any code changed. We can use -v option to mount the local folder as: +For building the wheel we use the ``cibuilwwheel`` utility, which can be installed using: -:: +.. code::sh - docker run -v /home/user/nmodl:/root/nmodl -it bluebrain/nmodl:wheel bash + pip install cibuildwheel -where ``/home/user/nmodl`` is the NMODL repository on the host machine. -We mount this directory inside docker at location ``/root/nmodl`` inside -the container. +Then to build a wheel for the current platform, run: -Note that for OS X there is no docker image but on a system where all -dependencies exist, you have to perform next building step. +.. code::sh -Launch the wheel building -------------------------- + cibuildwheel -Once we are inside docker container, we can start building wheels. There -is a build script that loops over the pythons ``>=3.5`` in -``/opt/python``, build and audit the generated wheels. Results are -placed in this wheelhouse directory. +If you have Docker installed, you can also build the Linux wheels using: -:: +.. code::sh - cd /root/nmodl - bash packaging/python/build_wheels.bash linux + cibuildwheel --platform linux -For OSX on a system with the all dependencies you have to clone the -NMODL repository and do: +Note that, if you happen to have Podman installed instead of Docker, you can +set ``CIBW_CONTAINER_ENGINE=podman`` to use Podman instead of Docker for this +task. -:: +Furthermore, in order to build wheels on MacOS, you need to install an official +CPython installer from `python.org `__. - cd nrn - bash packaging/python/build_wheels.bash osx +For a complete list of all available customization options for +``cibuildwheel``, please consult the +`documentation `__. -Updating neuron_wheel docker image ----------------------------------- +Testing the wheel +----------------- + +On MacOS, the testing of the wheel is launched automatically when running +``cibuildwheel``. On Linux, you will need to test the wheel manually, please +see :ref:`testing-installed-module` for the instructions. + + +Updating the NMODL Docker image +------------------------------- -If you have changed Dockerfile, you can build the new image as: +If you have changed the Dockerfile, you can build the new image as: -:: +.. code::sh docker build -t bluebrain/nmodl:tag . -and then push image to hub.docker.com as: +and then push the image to hub.docker.com as: -:: +.. code::sh docker login --username= - docker push bluebrain/nmodl:tag + docker push bluebrain/nmodl:tag diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash deleted file mode 100755 index 5dfb0fc493..0000000000 --- a/packaging/build_wheels.bash +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -set -xe -# A script to loop over the available pythons installed -# on Linux/OSX and build wheels -# -# Note: It should be invoked from nmodl directory -# -# PREREQUESITES: -# - cmake (>=3.15) -# - flex (>= 2.6) -# - bison (>=3.0) -# - python (>=3.8) -# - C/C++ compiler - -if ! [ -f pyproject.toml ]; then - echo "Error: pyproject.toml not found. Please launch $0 from the nmodl root dir" - exit 1 -fi - -setup_venv() { - local py_bin="$1" - local py_ver=$("$py_bin" -c "import sys; print('%d%d' % tuple(sys.version_info)[:2])") - local venv_dir="nmodl_build_venv$py_ver" - - if [ "$py_ver" -lt 37 ]; then - echo "[SKIP] Python $py_ver not supported" - skip=1 - return 0 - fi - - echo " - Creating $venv_dir: $py_bin -m venv $venv_dir" - "$py_bin" -m venv "$venv_dir" - . "$venv_dir/bin/activate" - - if ! pip install --upgrade pip setuptools wheel; then - curl https://bootstrap.pypa.io/get-pip.py | python - pip install --upgrade setuptools wheel - fi - -} - - -build_wheel_linux() { - echo "[BUILD WHEEL] Building with interpreter $1" - local skip= - setup_venv "$1" - (( $skip )) && return 0 - - echo " - Installing build requirements" - pip install pip auditwheel - - echo " - Building..." - rm -rf dist _build - # Workaround for https://github.com/pypa/manylinux/issues/1309 - git config --global --add safe.directory "*" - python -m pip wheel . --wheel-dir dist/ --no-deps - - echo " - Repairing..." - auditwheel repair dist/*.whl - - deactivate -} - - -build_wheel_osx() { - echo "[BUILD WHEEL] Building with interpreter $1" - local skip= - setup_venv "$1" - (( $skip )) && return 0 - - echo " - Installing build requirements" - pip install --upgrade delocate - - echo " - Building..." - rm -rf dist _build - # the custom `build-dir` is a workaround for this issue: - # https://gitlab.kitware.com/cmake/cmake/-/issues/20107 - python -m pip wheel . --wheel-dir dist/ -C build-dir=_build --no-deps - - echo " - Repairing..." - delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel - - deactivate -} - -# platform for which wheel we are building -platform="$1" - -# python version for which wheel we are building; 3* (default) means all python 3 versions -python_wheel_version="${2:-3*}" - -# MAIN - -case "${platform}" in - - linux) - python_wheel_version="${python_wheel_version//[-._]/}" - for py_bin in /opt/python/cp${python_wheel_version}-cp${python_wheel_version}/bin/python; do - build_wheel_linux "$py_bin" - done - ;; - - osx) - for py_bin in /Library/Frameworks/Python.framework/Versions/${python_wheel_version}/bin/python3; do - build_wheel_osx "$py_bin" - done - ;; - - *) - echo "Usage: $(basename "$0") [version]" - exit 1 - ;; - -esac diff --git a/pyproject.toml b/pyproject.toml index 0d057c2f46..7cf9f5353c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,21 @@ NMODL_BUILD_WHEEL = "ON" [tool.pytest.ini_options] testpaths = "test/unit/pybind" + +[tool.cibuildwheel] +skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64"] +test-extras = ["test"] +test-command = [ + "bash {package}/packaging/test_wheel.bash python3 {wheel} false", +] +manylinux-x86_64-image = "docker.io/bluebrain/nmodl:wheel" + +[tool.cibuildwheel.macos] +environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } +config-settings = {build-dir = "_build"} + +[tool.cibuildwheel.linux] +environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } +# the Linux wheel is not tested in cibuildwheel due to manylinux images not having +# libpython*.so, so this is tested manually in the CI +test-command = "true"