diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 323405636..f780e4d55 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,9 +19,12 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + ENV_FILE: envs/environment.yaml + jobs: - run: - name: Run + run-tests: + name: OS strategy: fail-fast: false matrix: @@ -29,14 +32,6 @@ jobs: - macos - windows - ubuntu - inhouse: - - stable-inhouse-deps - - dev-inhouse-deps - exclude: - - os: macos - inhouse: dev-inhouse-deps - - os: windows - inhouse: dev-inhouse-deps runs-on: ${{ matrix.os }}-latest defaults: @@ -46,57 +41,126 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup secrets + - name: Setup secrets & cache dates run: | echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc + echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts + echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env + + - uses: actions/cache@v4 + with: + path: | + data + cutouts + key: data-cutouts-${{ env.week }} + + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: pypsa-eur + + - name: Cache Conda env + uses: actions/cache@v4 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ env.today }}-${hashFiles('${{ env.ENV_FILE }}')} + id: cache-env + + - name: Update environment + if: steps.cache-env.outputs.cache-hit != 'true' + run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }} - - name: Set cache dates + - name: Run snakemake test workflows run: | + conda activate pypsa-eur + make test + + - name: Upload artifacts + uses: actions/upload-artifact@v3.2.1-node20 + with: + name: logs-${{ matrix.inhouse }} + path: | + logs + .snakemake/log + retention-days: 3 + + run-tests-on-dev-deps: + name: Inhouse + strategy: + fail-fast: false + matrix: + inhouse: + - pypsa + - atlite + - powerplantmatching + - linopy + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v4 + + # Only run checks if package is not pinned + - name: Check if inhouse package is pinned + run: | + grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.ENV_FILE }}) + if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then + echo "pinned=true" >> $GITHUB_ENV + else + echo "pinned=false" >> $GITHUB_ENV + fi + + - name: Setup secrets & cache dates + if: env.pinned == 'false' + run: | + echo -ne "url: ${CDSAPI_URL}\nkey: ${CDSAPI_TOKEN}\n" > ~/.cdsapirc echo "week=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_ENV # env - - name: Cache data and cutouts folders - uses: actions/cache@v4 + - uses: actions/cache@v4 + if: env.pinned == 'false' with: path: | data cutouts key: data-cutouts-${{ env.week }} - - name: Setup Conda - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@v3 + if: env.pinned == 'false' with: activate-environment: pypsa-eur - name: Cache Conda env + if: env.pinned == 'false' uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs - key: conda-${{ runner.os }}--${{ runner.arch }}--${{ env.today }}-${{ hashFiles('envs/environment.yaml') }} + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ matrix.inhouse }}-${{ env.today }}-${hashFiles('${{ env.ENV_FILE }}')} id: cache-env - name: Update environment - if: steps.cache-env.outputs.cache-hit != 'true' - run: conda env update -n pypsa-eur -f envs/environment.yaml + if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true' + run: conda env update -n pypsa-eur -f ${{ env.ENV_FILE }} - name: Install inhouse packages from master - if: matrix.inhouse == 'dev-inhouse-deps' + if: env.pinned == 'false' run: | - python -m pip install uv - uv pip install git+https://github.com/PyPSA/pypsa.git@master - uv pip install git+https://github.com/PyPSA/atlite.git@master - uv pip install git+https://github.com/PyPSA/powerplantmatching.git@master - uv pip install git+https://github.com/PyPSA/linopy.git@master + python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master - name: Run snakemake test workflows + if: env.pinned == 'false' run: | + conda activate pypsa-eur make test - name: Upload artifacts - if: matrix.os == 'ubuntu' && matrix.inhouse == 'stable-inhouse-deps' - uses: actions/upload-artifact@v4.4.0 + if: env.pinned == 'false' + uses: actions/upload-artifact@v4 with: - name: resources-results + name: logs-${{ matrix.inhouse }} path: | - results - retention-days: 7 + logs + .snakemake/log + retention-days: 3 diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index 38fa31abf..147d735f3 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -79,8 +79,8 @@ lines: solving: solver: - name: glpk - options: "glpk-default" + name: highs + options: highs-default plotting: diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 0ede9aa77..ba4c77795 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -86,8 +86,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 92379ae27..e14ff86ba 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -80,8 +80,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 781b3fd49..742b30124 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -85,8 +85,8 @@ industry: solving: solver: - name: glpk - options: glpk-default + name: highs + options: highs-default mem: 4000 plotting: diff --git a/config/test/config.scenarios.yaml b/config/test/config.scenarios.yaml index a9a826fdc..d238703db 100644 --- a/config/test/config.scenarios.yaml +++ b/config/test/config.scenarios.yaml @@ -57,5 +57,5 @@ renewable: solving: solver: - name: glpk - options: "glpk-default" + name: highs + options: highs-default