Merge branch 'nexus-upd' of https://github.com/iiasa/message-ix-models …

…into nexus-upd
iiasa · Apr 2, 2024 · 18b3648 · 18b3648
2 parents 63aad7c + 50e97e6
commit 18b3648
Show file tree

Hide file tree

Showing 75 changed files with 2,581 additions and 1,812 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,27 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    # Only update major versions
+    ignore:
+    - dependency-name: "*"
+      update-types:
+      - "version-update:semver-minor"
+      - "version-update:semver-patch"
+    # Below config mirrors the example at
+    # https://github.com/dependabot/dependabot-core/blob/main/.github/dependabot.yml
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+      time: "16:00"
+    groups:
+      all-actions:
+        patterns: [ "*" ]
+    assignees:
+    - "glatterf42"
+    - "khaeru"
+    labels:
+    - "dependencies"
+    reviewers:
+    - "glatterf42"
+    - "khaeru"
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -1,8 +1,6 @@
 name: Test
 
 on:
-  push:
-    branches: [ main ]
   pull_request:
     branches: [ main , "migrate-*"]
   schedule:
@@ -18,42 +16,65 @@ jobs:
   pytest:
     strategy:
       matrix:
-        # One job per OS; latest python version testable on GitHub actions.
-        # These should match the versions used in the "pytest" workflows of both
-        # ixmp and message_ix.
-        version:
-        - { os: macos-latest, python: "3.11" }
-        - { os: ubuntu-latest, python: "3.11" }
-        - { os: windows-latest, python: "3.11" }
-        # Versions of both ixmp and message_ix to use
+        os:
+        - macos-latest
+        - ubuntu-latest
+        - windows-latest
         upstream:
-        - { version: v3.4.0, extra-deps: '"pandas<2.0"' }  # Minimum version given in setup.cfg
-        - { version: v3.5.0, extra-deps: '"pandas<2.0"' }
-        - { version: v3.6.0, extra-deps: '"pandas<2.0"' }
-        - { version: v3.7.0, extra-deps: "" }  # Latest released version
-        - { version: main, extra-deps: "" }  # Development version
+        # In each group:
+        # - Versions of ixmp and message_ix to test.
+        # - Latest supported Python version for those or other dependencies.
+        # - Extra dependencies, in particular fixed/maximum versions to resolve conficts.
+        #   - dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0.
+        #     https://github.com/iiasa/message-ix-models/pull/156#issuecomment-2020152360
+        #   - genno: upstream versions < 3.8.0 import genno.computations, removed in 1.25.0.
+        #     https://github.com/iiasa/message-ix-models/pull/156
+        #   - pytest: upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0.
+        #     https://github.com/iiasa/message-ix-models/pull/155
+        #
+        # Minimum version given in pyproject.toml
+        - version: v3.4.0
+          python-version: "3.11"
+          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
+        - version: v3.5.0
+          python-version: "3.11"
+          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
+        - version: v3.6.0
+          python-version: "3.11"
+          extra-deps: '"dask < 2024.3.0" "genno < 1.25" "pandas < 2.0" "pytest == 8.0.0"' #
+        - version: v3.7.0
+          python-version: "3.11"
+          extra-deps: 'dask[dataframe]   "genno < 1.25"                "pytest == 8.0.0"' #
+        # Latest released version
+        - version: v3.8.0
+          python-version: "3.12"
+          extra-deps: 'dask[dataframe]                                 "pytest == 8.0.0"' #
+        # Development version
+        - version: main
+          python-version: "3.12"
+          extra-deps: 'dask[dataframe]' #
 
       fail-fast: false
 
-    runs-on: ${{ matrix.version.os }}
-    name: ${{ matrix.version.os }}-py${{ matrix.version.python }}-upstream-${{ matrix.upstream.version }}
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }}-py${{ matrix.upstream.python-version }}-upstream-${{ matrix.upstream.version }}
 
     steps:
     - name: Cache test data
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: message-local-data
-        key: ${{ matrix.version.os }}
+        key: ${{ matrix.os }}
 
     - name: Check out message-ix-models
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         lfs: true
         fetch-depth: ${{ env.depth }}
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.version.python }}
+        python-version: ${{ matrix.upstream.python-version }}
         cache: pip
         cache-dependency-path: "**/pyproject.toml"
 
@@ -62,7 +83,7 @@ jobs:
         version: 25.1.1
         license: ${{ secrets.GAMS_LICENSE }}
 
-    - uses: ts-graphviz/setup-graphviz@v1
+    - uses: ts-graphviz/setup-graphviz@v2
       with:
         macos-skip-brew-update: true
 
@@ -98,21 +119,23 @@ jobs:
       shell: bash
 
     - name: Upload test coverage to Codecov.io
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }} # required
 
   pre-commit:
     name: Code quality
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with: { python-version: "3.11" }
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with: { python-version: "3.12" }
 
     - name: Force recreation of pre-commit virtual environment for mypy
       if: github.event_name == 'schedule'  # Comment this line to run on a PR
       run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true
       env: { GH_TOKEN: "${{ github.token }}" }
 
-    - uses: pre-commit/[email protected].0
+    - uses: pre-commit/[email protected].1
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,9 +8,9 @@ repos:
     pass_filenames: false
 
     language: python
-    entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@"
+    entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@; python -m pip list"
     additional_dependencies:
-    - mypy >= 1.8.0
+    - mypy >= 1.9.0
     - plotnine
     - pytest
     - sdmx1
@@ -20,7 +20,7 @@ repos:
     - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
     args: ["."]
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.1.13
+  rev: v0.3.2
   hooks:
   - id: ruff
   - id: ruff-format

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,4 +3,8 @@ prune message_ix_models/data/test/iea
 prune message_ix_models/data/test/MESSAGEix-GLOBIOM_1.1_R11_no-policy_baseline
 prune message_ix_models/data/test/ssp
 prune message_ix_models/data/water/*
+# Larger package data
+# - Not distributed on PyPI.
+# - Should be fetched with Pooch from GitHub.
+exclude message_ix_models/data/ssp/*.gz
 exclude message_ix_models/data/water/*.tar.xz
diff --git a/doc/api/util.rst b/doc/api/util.rst
@@ -34,6 +34,7 @@ Commonly used:
    check_support
    convert_units
    copy_column
+   datetime_now_with_tz
    ffill
    identify_nodes
    iter_keys
@@ -52,6 +53,7 @@ Commonly used:
    same_node
    same_time
    series_of_pint_quantity
+   show_versions
 
 .. automodule:: message_ix_models.util
    :members: