From 0b0f8fc9d1cf368dd5b9ec71761ca64e6ded6c12 Mon Sep 17 00:00:00 2001 From: Lisandro Dalcin Date: Thu, 5 Oct 2023 14:16:28 +0300 Subject: [PATCH] Test Linux wheels in containers --- .cibw/test-linux-debian.sh | 16 ++++++ .cibw/test-linux-fedora.sh | 25 +++++++++ .cibw/test-linux-ubuntu.sh | 17 ++++++ .cibw/test-macos.sh | 23 ++++++++ .cibw/test-wheel.sh | 80 ++++++++++++++++++++++++++++ .github/workflows/cd-wheel.yml | 96 +++++++++++++++++++++++++++++++++- 6 files changed, 255 insertions(+), 2 deletions(-) create mode 100755 .cibw/test-linux-debian.sh create mode 100755 .cibw/test-linux-fedora.sh create mode 100755 .cibw/test-linux-ubuntu.sh create mode 100755 .cibw/test-macos.sh create mode 100755 .cibw/test-wheel.sh diff --git a/.cibw/test-linux-debian.sh b/.cibw/test-linux-debian.sh new file mode 100755 index 0000000..dc80f32 --- /dev/null +++ b/.cibw/test-linux-debian.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail + +packages=( + python3.11-venv + pypy3 + libmpich12 + libopenmpi3 +) +sudo=$(command -v sudo || true) +$sudo apt update -y +$sudo apt install -y ${packages[@]} +$sudo ln -sr /usr/lib/$(arch)-linux-gnu/libmpi{ch,}.so.12 + +sdir=$(cd "$(dirname -- "$0")" && pwd -P) +$sdir/test-wheel.sh diff --git a/.cibw/test-linux-fedora.sh b/.cibw/test-linux-fedora.sh new file mode 100755 index 0000000..e2f398e --- /dev/null +++ b/.cibw/test-linux-fedora.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +packages=( + python3.6 + python3.7 + python3.8 + python3.9 + python3.10 + python3.11 + python3.12 + pypy3.9 + pypy3.10 + mpich + openmpi +) +sudo=$(command -v sudo || true) +opts=--setopt=install_weak_deps=False +$sudo dnf install -y $opts ${packages[@]} +set +u +source /etc/profile.d/modules.sh +set -u + +sdir=$(cd "$(dirname -- "$0")" && pwd -P) +$sdir/test-wheel.sh diff --git a/.cibw/test-linux-ubuntu.sh b/.cibw/test-linux-ubuntu.sh new file mode 100755 index 0000000..c71a757 --- /dev/null +++ b/.cibw/test-linux-ubuntu.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -euo pipefail + +packages=( + python3.10-venv + python3.11-venv + pypy3 + libmpich12 + libopenmpi3 +) +sudo=$(command -v sudo || true) +$sudo apt update -y +$sudo apt install -y ${packages[@]} +$sudo ln -sr /usr/lib/$(arch)-linux-gnu/libmpi{ch,}.so.12 + +sdir=$(cd "$(dirname -- "$0")" && pwd -P) +$sdir/test-wheel.sh diff --git a/.cibw/test-macos.sh b/.cibw/test-macos.sh new file mode 100755 index 0000000..7ad316e --- /dev/null +++ b/.cibw/test-macos.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +packages_py=( + python@3.8 + python@3.9 + python@3.10 + python@3.11 + pypy3.9 + pypy3.10 +) +packages_mpi=( + mpich + openmpi +) +brew install ${packages_py[@]} +for mpi in ${packages_mpi[@]}; do + brew install $mpi + brew unlink $mpi +done + +sdir=$(cd "$(dirname -- "$0")" && pwd -P) +$sdir/test-wheel.sh diff --git a/.cibw/test-wheel.sh b/.cibw/test-wheel.sh new file mode 100755 index 0000000..00f61f9 --- /dev/null +++ b/.cibw/test-wheel.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -euo pipefail + +PYTHON=( + python3.6 + python3.7 + python3.8 + python3.9 + python3.10 + python3.11 + python3.12 + pypy3.7 + pypy3.8 + pypy3.9 +) + +wheelhouse=${1:-dist} +venvroot=$(mktemp -d) +trap "rm -rf $venvroot" EXIT +export PIP_QUIET=1 + +if [[ $(uname) =~ NT ]]; then + MPI=(impi msmpi) + bin=Scripts +else + MPI=(mpich openmpi) + bin=bin +fi + +function setup-python { + local python=$1 + venvdir=$venvroot/$python + $python -m venv $venvdir + source $venvdir/$bin/activate + python -m pip install pip --upgrade +} + +function setup-mpi4py { + local dist=$1 + python -m pip install mpi4py --no-index --find-links=$dist +} + +function setup-mpi { + local mpi=$1 + if command -v brew > /dev/null; then + brew unlink mpich openmpi > /dev/null + brew link $mpi > /dev/null + elif command -v module > /dev/null; then + module unload mpi + module load mpi/$mpi-$(arch) + else + export MPI4PY_MPIABI=$mpi + fi +} + +function mpi4py-test-basic { + python --version + python -m mpi4py --version + python -m mpi4py --mpi-std-version + python -m mpi4py --mpi-lib-version | head -n 1 + echo +} + +for python in ${PYTHON[@]}; do + test -z "$(command -v $python)" && continue + setup-python $python + setup-mpi4py $wheelhouse + for mpi in ${MPI[@]}; do + setup-mpi $mpi + mpi4py-test-basic + export MPI4PY_MPIABI=$mpi + mpi4py-test-basic + if [ $mpi == mpich -o $mpi == openmpi ]; then + export MPI4PY_MPIABI=mpi31-$mpi + mpi4py-test-basic + fi + unset MPI4PY_MPIABI + done + deactivate destructive +done diff --git a/.github/workflows/cd-wheel.yml b/.github/workflows/cd-wheel.yml index 8bfbe0f..550ed6c 100644 --- a/.github/workflows/cd-wheel.yml +++ b/.github/workflows/cd-wheel.yml @@ -278,8 +278,100 @@ jobs: - id: setup run: python -u .cibw/setup-test.py wheelhouse >> "$GITHUB_OUTPUT" + test-Linux: + if: ${{ contains(needs.setup.outputs.os-arch-list, '"Linux-x86_64"') }} + needs: [setup, merge] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: + - ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: wheel-${{ runner.os }} + path: dist + + - uses: docker://debian:latest + with: + entrypoint: /bin/bash + args: .cibw/test-linux-debian.sh + + - uses: docker://ubuntu:latest + with: + entrypoint: /bin/bash + args: .cibw/test-linux-ubuntu.sh + + - uses: docker://fedora:latest + with: + entrypoint: /bin/bash + args: .cibw/test-linux-fedora.sh + + test-macOS: + if: ${{ contains(needs.setup.outputs.os-arch-list, '"macOS-x86_64"') }} + needs: [setup, merge] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: + - macos-11 + - macos-12 + - macos-13 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: wheel-${{ runner.os }} + path: dist + + - run: .cibw/test-macos.sh + shell: bash + + test-Windows: + if: ${{ contains(needs.setup.outputs.os-arch-list, '"Windows-AMD64"') }} + needs: [setup, merge] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: + - windows-2019 + - windows-2022 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: wheel-${{ runner.os }} + path: dist + + - uses: actions/setup-python@v4 + with: + python-version: | + pypy-3.8 + pypy-3.9 + 3.7 + 3.8 + 3.9 + 3.10 + 3.11 + 3.12 + + - uses: mpi4py/setup-mpi@v1 + with: + mpi: intelmpi + - uses: mpi4py/setup-mpi@v1 + with: + mpi: msmpi + + - run: .cibw/test-wheel.sh + shell: bash + test-cf: - if: ${{ needs.test-setup.outputs.matrix-test-cf != '[]' }} + if: ${{ false }} # ${{ needs.test-setup.outputs.matrix-test-cf != '[]' }} needs: test-setup runs-on: ${{ (matrix.os == 'Linux' && 'ubuntu-latest' ) || (matrix.os == 'macOS' && 'macos-latest' ) || @@ -358,7 +450,7 @@ jobs: shell: bash -el {0} test-gh: - if: ${{ needs.test-setup.outputs.matrix-test-gh != '[]' }} + if: ${{ false }} # ${{ needs.test-setup.outputs.matrix-test-gh != '[]' }} needs: test-setup runs-on: ${{ matrix.runner }} strategy: