Test workflows #44
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors | |
| # | |
| # SPDX-License-Identifier: CC0-1.0 | |
| name: Test workflows | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| schedule: | |
| - cron: "0 5 * * *" | |
| # Cancel any in-progress runs when a new run is triggered | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| ENV_FILE: envs/environment.yaml | |
| ENV_FIXED_FILE: envs/environment.fixed.yaml | |
| jobs: | |
| run-tests: | |
| name: OS | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - macos | |
| - windows | |
| - ubuntu | |
| runs-on: ${{ matrix.os }}-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup secrets & cache dates | |
| run: | | |
| echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc | |
| echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts | |
| echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| data | |
| cutouts | |
| key: data-cutouts-${{ env.week }} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: pypsa-eur | |
| - name: Cache Conda env | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: conda-${{ runner.os }}-${{ runner.arch }}-${{ env.today }}-${{ hashFiles(format('{0}', env.ENV_FILE)) }} | |
| id: cache-env | |
| - name: Update environment | |
| if: steps.cache-env.outputs.cache-hit != 'true' | |
| run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }} | |
| - name: Log env diff to environment.fixed.yaml | |
| if: ${{ matrix.os == 'ubuntu' }} | |
| run: | | |
| # Get fixed environment of current env | |
| conda env export --name ${{ github.event.repository.name }} --no-builds | sed 's/^name: ${{ github.event.repository.name }}$/name: ${{ github.event.repository.name }}-fixed/' > current-env.yaml | |
| # Add SPDX header | |
| SPDX_HEADER="# SPDX-FileCopyrightText: : 2017-2024 The PyPSA-Eur Authors\n# SPDX-License-Identifier: CC0-1.0\n" | |
| echo -e "$SPDX_HEADER" | cat - current-env.yaml > temp && mv temp current-env.yaml | |
| # Format with pre-commit (it differs from the conda output) | |
| pip install pre-commit | |
| pre-commit run --files current-env.yaml || true | |
| # Get diff | |
| diff ${{ env.ENV_FIXED_FILE }} current-env.yaml > diff.txt || true | |
| # Format | |
| { echo -e "**Environment diff**\n\`\`\` diff\n"; cat diff.txt; echo -e "\n\`\`\`"; } > temp && mv temp diff.txt | |
| # Write to summary | |
| cat diff.txt >> $GITHUB_STEP_SUMMARY | |
| shell: bash | |
| - name: Run snakemake test workflows | |
| run: | | |
| make test | |
| - name: Upload artifacts | |
| uses: actions/[email protected] | |
| with: | |
| name: logs-${{ matrix.inhouse }} | |
| path: | | |
| logs | |
| .snakemake/log | |
| retention-days: 3 | |
| run-tests-on-dev-deps: | |
| name: Inhouse | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| inhouse: | |
| - pypsa | |
| - atlite | |
| - powerplantmatching | |
| - linopy | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Only run checks if package is not pinned | |
| - name: Check if inhouse package is pinned | |
| run: | | |
| grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.ENV_FILE }}) | |
| if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then | |
| echo "pinned=true" >> $GITHUB_ENV | |
| else | |
| echo "pinned=false" >> $GITHUB_ENV | |
| fi | |
| - name: Setup secrets & cache dates | |
| if: env.pinned == 'false' | |
| run: | | |
| echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc | |
| echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts | |
| echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env | |
| - uses: actions/cache@v4 | |
| if: env.pinned == 'false' | |
| with: | |
| path: | | |
| data | |
| cutouts | |
| key: data-cutouts-${{ env.week }} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| if: env.pinned == 'false' | |
| with: | |
| activate-environment: pypsa-eur | |
| - name: Cache Conda env | |
| if: env.pinned == 'false' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: conda-${{ runner.os }}-${{ runner.arch }}-${{ matrix.inhouse }}-${{ env.today }}-${{ hashFiles(format('{0}', env.ENV_FILE)) }} | |
| id: cache-env | |
| - name: Update environment | |
| if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true' | |
| run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }} | |
| - name: Install inhouse packages from master | |
| if: env.pinned == 'false' | |
| run: | | |
| python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master | |
| - name: Run snakemake test workflows | |
| if: env.pinned == 'false' | |
| run: | | |
| make test | |
| - name: Upload artifacts | |
| if: env.pinned == 'false' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: logs-${{ matrix.inhouse }} | |
| path: | | |
| logs | |
| .snakemake/log | |
| retention-days: 3 |