pytest.yaml

name: Test

on:
  pull_request:
    branches: [ main , "migrate**"]
  schedule:
  # 05:00 UTC = 06:00 CET = 07:00 CEST
  - cron: "0 5 * * *"

# Cancel previous runs that have not completed
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  warm-lfs-cache:
    strategy:
      matrix:
        os: [ macos-13, macos-latest, ubuntu-latest, windows-latest ]
    runs-on: ${{ matrix.os }}
    steps:
    - uses: francisbilham11/action-cached-lfs-checkout@v3

  pytest:
    needs: warm-lfs-cache

    strategy:
      matrix:
        os:
        - macos-13
        - macos-latest
        - ubuntu-latest
        - windows-latest
        version:
        # In each group:
        # - Versions of ixmp and message_ix to test.
        # - Latest supported Python version for those or other dependencies.
        # Minimum version given in pyproject.toml + earlier version of Python
        - { upstream: v3.4.0, python: "3.11" }  # 2022-01-27
        - { upstream: v3.5.0, python: "3.11" }  # 2022-05-06
        - { upstream: v3.6.0, python: "3.11" }  # 2022-08-18
        - { upstream: v3.7.0, python: "3.11" }  # 2023-05-17
        - { upstream: v3.8.0, python: "3.12" }  # 2024-01-12
        # Latest released version + latest released Python
        - { upstream: v3.9.0, python: "3.13" }  # 2024-06-04
        # Development version + latest released Python
        - { upstream: main,   python: "3.13" }

        exclude:
        # Specific version combinations that are invalid / not to be used
        # These versions of ixmp are not able locate the arm64 GAMS API binaries
        - { os: macos-latest, version: {upstream: v3.4.0 }}
        - { os: macos-latest, version: {upstream: v3.5.0 }}
        - { os: macos-latest, version: {upstream: v3.6.0 }}
        - { os: macos-latest, version: {upstream: v3.7.0 }}
        - { os: macos-latest, version: {upstream: v3.8.0 }}
        - { os: macos-latest, version: {upstream: v3.9.0 }}
        # Redundant with macos-latest
        - { os: macos-13, version: {upstream: main }}

      fail-fast: false

    runs-on: ${{ matrix.os }}
    name: ${{ matrix.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.version.upstream }}

    steps:
    - name: Cache test data
      uses: actions/cache@v4
      with:
        path: message-local-data
        key: ${{ matrix.os }}

    - name: Check out message-ix-models
      uses: francisbilham11/action-cached-lfs-checkout@v3

    - uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.version.python }}
        cache: pip
        cache-dependency-path: "**/pyproject.toml"

    - uses: iiasa/actions/setup-gams@main
      with:
        version: 43.4.1
        license: ${{ secrets.GAMS_LICENSE }}

    - uses: ts-graphviz/setup-graphviz@v2
      # Work around ts-graphviz/setup-graphviz#630
      if: ${{ matrix.os != 'macos-13' }}

    - name: Determine extra dependencies
      id: extra-deps
      run : |
        from os import environ
        from pathlib import Path

        v, result = "${{ matrix.version.upstream }}".replace("main", "vmain"), []
        for condition, dependency in (
            (v <= "v3.6.0", "dask < 2024.3.0"),  # dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0 (#156)
            (v <= "v3.6.0", "pandas < 2.0"),
            (v >= "v3.7.0", "dask[dataframe] < 2024.11.0"),  # dask >= 2024.11.0 changes handling of dict (will be addressed in #225)
            (v <= "v3.7.0", "genno < 1.25"),  # Upstream versions < 3.8.0 import genno.computations, removed in 1.25.0 (#156)
            (v < "v3.9.0", "pytest == 8.0.0"),  # Upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0 (#155)
        ):
            result.extend([f'"{dependency}"'] if condition else [])

        Path(environ["GITHUB_OUTPUT"]).write_text(f"value={' '.join(result)}\n")
      shell: python

    - name: Install packages and dependencies
      # By default, install:
      # - ixmp, message_ix: from GitHub branches/tags per matrix.version.upstream (above)
      # - other dependencies including genno: from PyPI.
      #
      # To test against unreleased code (on `main`, or other branches
      # for open PRs), temporarily uncomment, add, or edit lines below
      # as needed. DO NOT merge such changes to `main`.
      run: |
        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.version.upstream }}"
        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.version.upstream }}"

        pip install .[docs,tests] ${{ steps.extra-deps.outputs.value }}

        # TEMPORARY With Python 3.13 pyam-iamc resolves to 1.3.1, which in turn
        # limits pint < 0.17. Override.
        pip install --upgrade pint

    - name: Configure local data path
      run: |
        mkdir -p message-local-data/cache
        mix-models config set "message local data" "$(realpath message-local-data)"
        mix-models config show
      shell: bash

    - name: Run test suite using pytest
      run: |
        pytest message_ix_models \
          -m "not (ece_db or snapshot)" \
          -rA --verbose --color=yes --durations=20 \
          --cov-report=term-missing --cov-report=xml \
          --numprocesses=auto \
          --local-cache --jvmargs="-Xmx6G"
      shell: bash

    - name: Upload test coverage to Codecov.io
      uses: codecov/codecov-action@v5
      with:
        token: ${{ secrets.CODECOV_TOKEN }} # required

  pre-commit:
    name: Code quality

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v5
      with: { python-version: "3.13" }

    - name: Force recreation of pre-commit virtual environment for mypy
      if: github.event_name == 'schedule'  # Comment this line to run on a PR
      run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true
      env: { GH_TOKEN: "${{ github.token }}" }

    - uses: pre-commit/action@v3.0.1