From c90a1277db13156d245efe3f1172019caa6f2561 Mon Sep 17 00:00:00 2001 From: Lisandro Dalcin Date: Sun, 6 Aug 2023 14:58:54 +0300 Subject: [PATCH] GHA: Fix CI issues, setup dependabot, setup Codecov --- .github/dependabot.yml | 6 +++ .github/workflows/main.yml | 86 +++++++++++++++++++++++++++++--------- .gitignore | 1 - azure-pipelines.yml | 75 ++++++++++++++++++--------------- conf/requirements-test.txt | 3 ++ tests/runtests.sh | 64 ++++++++++++---------------- tests/test_mpifft.py | 4 +- tests/test_speed.py | 7 +--- 8 files changed, 146 insertions(+), 100 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 conf/requirements-test.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0dbe114..681c8fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,29 +1,77 @@ -name: github-CI +name: main -on: [push] +on: + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: jobs: - build: - + test: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - name: Python ${{ matrix.python-version }} example + os: + - ubuntu-latest + - macos-latest + mpi: + - mpich + - openmpi + py: + # - "3.7" + # - "3.8" + # - "3.9" + - "3.10" + - "3.11" + - "3.12" steps: - - uses: actions/checkout@v3 - - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* - - name: setup-conda - uses: s-weigand/setup-conda@v1 + + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 with: - update-conda: true - python-version: ${{ matrix.python-version }} - conda-channels: anaconda, conda-forge - - run: conda --version - - run: which python + python-version: ${{ matrix.py }} + - run: | - conda config --set always_yes yes - conda install -n root conda-build numpy fftw - conda build ./conf + # Install fftw + case $(uname) in + Linux) + sudo apt update + sudo apt install -y -q libfftw3-dev + ;; + Darwin) + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + brew install fftw + ;; + esac + + - run: python -m pip install -U pip build + + - run: python -m build + + - uses: mpi4py/setup-mpi@v1 + with: + mpi: ${{ matrix.mpi }} + + - run: pip install -vvv dist/mpi4py_fft-*.whl + env: + CFLAGS: "-O0" + + - run: pip install -r conf/requirements-test.txt + + - if: matrix.mpi == 'mpich' && startsWith(matrix.os, 'ubuntu') + run: ./runtests.sh + working-directory: tests + + - if: matrix.mpi == 'mpich' && startsWith(matrix.os, 'ubuntu') + uses: codecov/codecov-action@v4 + with: + files: test/coverage.xml + name: ${{ matrix.os }}-${{ matrix.mpi }}-${{ matrix.py }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 1d4d981..5c9f760 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ _templates/ *.dat *.html *.xml -*.txt *.h5 *.nc *.xdmf diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4d8c6d2..5202d45 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,35 +1,35 @@ jobs: -- job: macOS - displayName: macos-latest - pool: - vmImage: 'macos-latest' - strategy: - matrix: - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - - steps: - - bash: echo "##vso[task.prependpath]$CONDA/bin" - displayName: Add conda to PATH - - - bash: sudo chown -R $USER $CONDA - displayName: Take ownership of conda installation - - - bash: | - conda config --add channels conda-forge - conda config --set always_yes yes - conda install -n root conda-build numpy fftw - displayName: Set up Anaconda - - - bash: | - conda build --python $PYTHON_VERSION ./conf - displayName: Build and test +# - job: macOS +# displayName: macos-latest +# pool: +# vmImage: 'macos-latest' +# strategy: +# matrix: +# Python37: +# python.version: '3.7' +# Python38: +# python.version: '3.8' +# Python39: +# python.version: '3.9' +# Python310: +# python.version: '3.10' +# +# steps: +# - bash: echo "##vso[task.prependpath]$CONDA/bin" +# displayName: Add conda to PATH +# +# - bash: sudo chown -R $USER $CONDA +# displayName: Take ownership of conda installation +# +# - bash: | +# conda config --add channels conda-forge +# conda config --set always_yes yes +# conda install -n root conda-build numpy fftw +# displayName: Set up Anaconda +# +# - bash: | +# conda build --python $PYTHON_VERSION ./conf +# displayName: Build and test - job: Ubuntu displayName: Ubuntu @@ -45,6 +45,10 @@ jobs: python.version: '3.9' Python310: python.version: '3.10' + Python311: + python.version: '3.11' + Python312: + python.version: '3.12' steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" @@ -53,13 +57,16 @@ jobs: - bash: | conda config --add channels conda-forge conda config --set always_yes yes - conda install -n root conda-build numpy fftw + conda install -n root mpi4py mpich numpy fftw displayName: Set up Anaconda - bash: | - conda build --no-test --python $PYTHON_VERSION ./conf - conda create --name mpi4py_fft_env mpi4py_fft_test coverage scipy pyfftw=0.12 python=$PYTHON_VERSION --use-local + conda create --name mpi4py_fft_env \ + python=$PYTHON_VERSION \ + mpi4py mpich numpy fftw \ + pyfftw scipy coverage source activate mpi4py_fft_env + pip install -vvv --no-deps . pip install codecov cd tests && ./runtests.sh displayName: Build and test diff --git a/conf/requirements-test.txt b/conf/requirements-test.txt new file mode 100644 index 0000000..8eadf40 --- /dev/null +++ b/conf/requirements-test.txt @@ -0,0 +1,3 @@ +coverage +pyfftw; sys_platform == 'linux' and python_version < '3.12' +scipy diff --git a/tests/runtests.sh b/tests/runtests.sh index 341d36e..d1417d7 100755 --- a/tests/runtests.sh +++ b/tests/runtests.sh @@ -1,47 +1,35 @@ #!/bin/sh set -e -PY=$(python -c 'import sys; print(sys.version_info.major)') -export OMPI_MCA_plm=isolated -export OMPI_MCA_btl_vader_single_copy_mechanism=none -export OMPI_MCA_rmaps_base_oversubscribe=yes +export OMPI_MCA_plm_ssh_agent=false +export OMPI_MCA_pml=ob1 +export OMPI_MCA_btl=tcp,self +export OMPI_MCA_mpi_yield_when_idle=true +export OMPI_MCA_btl_base_warn_component_unused=false +export OMPI_MCA_rmaps_base_oversubscribe=true +export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe -if [ $PY -eq 3 ]; then - # coverage only for python version 3 +python -m coverage erase - python -m coverage erase +python -m coverage run -m test_fftw +python -m coverage run -m test_libfft +python -m coverage run -m test_io +python -m coverage run -m test_darray - python -m coverage run -m test_fftw - python -m coverage run -m test_libfft - python -m coverage run -m test_io - python -m coverage run -m test_darray - mpiexec -n 2 python -m coverage run -m test_pencil +mpiexec -n 2 python -m coverage run -m test_pencil +#mpiexec -n 4 python -m coverage test_pencil.py +#mpiexec -n 8 python -m coverage test_pencil.py - #mpiexec -n 4 python -m coverage test_pencil.py - #mpiexec -n 8 python -m coverage test_pencil.py - mpiexec -n 2 python -m coverage run -m test_mpifft - #mpiexec -n 4 python -m coverage test_mpifft.py - # mpiexec -n 8 python -m coverage test_mpifft.py - # mpiexec -n 12 python -m coverage test_mpifft.py - mpiexec -n 2 python -m coverage run spectral_dns_solver.py - mpiexec -n 2 python -m coverage run -m test_io - #mpiexec -n 4 python -m coverage run -m test_io - mpiexec -n 2 python -m coverage run -m test_darray - #mpiexec -n 4 python -m coverage run -m test_darray +mpiexec -n 2 python -m coverage run -m test_mpifft +#mpiexec -n 4 python -m coverage test_mpifft.py +#mpiexec -n 8 python -m coverage test_mpifft.py +#mpiexec -n 12 python -m coverage test_mpifft.py +mpiexec -n 2 python -m coverage run spectral_dns_solver.py - python -m coverage combine +mpiexec -n 2 python -m coverage run -m test_io +#mpiexec -n 4 python -m coverage run -m test_io +mpiexec -n 2 python -m coverage run -m test_darray +#mpiexec -n 4 python -m coverage run -m test_darray -else - python test_fftw.py - python test_libfft.py - mpiexec -n 2 python test_pencil.py - #mpiexec -n 4 python test_pencil.py - #mpiexec -n 8 python test_pencil.py - mpiexec -n 2 python test_mpifft.py - #mpiexec -n 4 python test_mpifft.py - # mpiexec -n 8 python test_mpifft.py - # mpiexec -n 12 python test_mpifft.py - mpiexec -n 2 python test_io.py - mpiexec -n 2 python test_darray.py - mpiexec -n 2 python spectral_dns_solver.py -fi +python -m coverage combine +python -m coverage xml diff --git a/tests/test_mpifft.py b/tests/test_mpifft.py index 2ea851a..5d4ca30 100644 --- a/tests/test_mpifft.py +++ b/tests/test_mpifft.py @@ -152,8 +152,8 @@ def test_mpifft(): fft.backward.input_array.shape) assert (fft.backward.output_pencil.subshape == fft.backward.output_array.shape) - assert np.alltrue(np.array(fft.global_shape(True)) == np.array(fft.forward.output_pencil.shape)) - assert np.alltrue(np.array(fft.global_shape(False)) == np.array(fft.forward.input_pencil.shape)) + assert np.all(np.array(fft.global_shape(True)) == np.array(fft.forward.output_pencil.shape)) + assert np.all(np.array(fft.global_shape(False)) == np.array(fft.forward.input_pencil.shape)) ax = -1 if axes is None else axes[-1] if isinstance(axes[-1], int) else axes[-1][-1] assert fft.forward.input_pencil.substart[ax] == 0 assert fft.backward.output_pencil.substart[ax] == 0 diff --git a/tests/test_speed.py b/tests/test_speed.py index 5d20553..ce70fa0 100644 --- a/tests/test_speed.py +++ b/tests/test_speed.py @@ -1,15 +1,10 @@ from time import time import numpy as np +import pyfftw import scipy.fftpack as sp from mpi4py_fft import fftw import pickle -try: - import pyfftw -except ImportError: - print('pyFFTW not available') - raise SystemExit(0) - try: #fftw.import_wisdom('wisdom.dat') pyfftw.import_wisdom(pickle.load(open('pyfftw.wisdom', 'rb')))