Update CI and tests

.github/workflows/publish.yaml

@@ -13,41 +13,6 @@ on:
 
 jobs:
   publish:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-
-    - uses: actions/setup-python@v2
-
-    - name: Cache Python packages
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cache/pip
-        key: publish-${{ runner.os }}
-
-    - name: Upgrade pip, wheel, setuptools-scm
-      run: python -m pip install --upgrade pip wheel setuptools-scm twine
-
-    - name: Build package
-      run: |
-        python3 setup.py bdist_wheel sdist
-        twine check dist/*
-
-    - name: Publish to TestPyPI
-      uses: pypa/[email protected]
-      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-      with:
-        user: __token__
-        password: ${{ secrets.TESTPYPI_TOKEN }}
-        repository_url: https://test.pypi.org/legacy/
-
-    - name: Publish to PyPI
-      uses: pypa/[email protected]
-      if: github.event_name == 'release'
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_TOKEN }}
+    uses: iiasa/actions/.github/workflows/publish.yaml@main
+    secrets:
+      PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.github/workflows/pytest.yaml

@@ -1,4 +1,4 @@
-name: Test suite & diagnostics
+name: Test
 
 on:
   push:
@@ -8,34 +8,37 @@ on:
   schedule:
   - cron: "0 5 * * *"
 
+# Cancel previous runs that have not completed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   # Location of RCLONE configuration file
   RCLONE_CONFIG: ci/rclone.conf
   # Path & URL fragments for uploaded historical data & diagnostics
   gcs_bucket: gcs:data.transportenergy.org/historical/ci/
   gcs_url: https://storage.googleapis.com/data.transportenergy.org/historical/ci/
-  # True if the event is a pull request and the incoming branch is within the
-  # transportenergy/database repo (as opposed to a fork). Only under this
-  # condition is the GCS_SERVICE_ACCOUNT_* secret available.
-  pr_from_main_repo: github.event_name != 'pull_request' || startsWith(github.event.pull_request.head.label, 'transportenergy:')
 
 jobs:
   pytest:
     strategy:
       matrix:
-        include:
-        - os: ubuntu-latest
-          python-version: "3.7"
-          run-diagnostics: false
-        - os: ubuntu-latest
-          python-version: "3.8"
-          run-diagnostics: false
-        - os: ubuntu-latest
-          python-version: "3.9"
-          run-diagnostics: true
-        - os: windows-latest
-          python-version: "3.9"
-          run-diagnostics: false
+        os:
+        - macos-latest
+        - ubuntu-latest
+        - windows-latest
+
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
+
+        # TEMPORARY Never run diagnostics
+        run-diagnostics:
+        - false
 
       fail-fast: false
 
@@ -44,67 +47,92 @@ jobs:
     name: ${{ matrix.os }}-py${{ matrix.python-version }}
 
     steps:
-    - name: Cancel previous runs that have not completed
-      uses: styfle/[email protected]
-      with:
-        access_token: ${{ github.token }}
-
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: true
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: "**/pyproject.toml"
 
-    - name: Cache Python packages
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cache/pip
-          ~/.cache/rclone*.zip
-          ~/appdata/local/pip/cache
-        key: ${{ matrix.os }}-${{ matrix.python-version }}
+    - name: Upgrade pip
+      run: python -m pip install --upgrade pip
 
-    - name: Upgrade pip, wheel
-      run: python -m pip install --upgrade pip wheel
-
-    - name: Install the Python package and dependencies
-      run: pip install --editable .[tests]
+    - name: Install the Python package and its dependencies
+      run: pip install .[tests]
 
     - name: Run pytest
       env:
         OK_API_KEY: ${{ secrets.OPENKAPSARC_API_KEY }}
       run: pytest --color=yes --cov-report=xml --verbose item
 
     - name: Upload test coverage to Codecov.io
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v3
+
+  diagnostics:
+    # Temporarily disabled to merge #84
+    if: false
+    # True if the event is a pull request and the incoming branch is within the
+    # transportenergy/database repo (as opposed to a fork). Only under this
+    # condition is the GCS_SERVICE_ACCOUNT_* secret available.
+    # if: github.event_name != 'pull_request' || startsWith(github.event.pull_request.head.label, 'transportenergy:')
+
+    needs: [pytest]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - uses: actions/setup-python@v4
+      with:  # Should be the same as the latest supported version, above
+        python-version: "3.12"
+        cache: pip
+        cache-dependency-path: "**/pyproject.toml"
+
+    - name: Upgrade pip
+      run: python -m pip install --upgrade pip
 
-    - name: Install rclone
-      if: env.pr_from_main_repo && matrix.run-diagnostics
+    - name: Install the Python package and its dependencies
+      run: pip install .[tests]
+
+    - name: Set up Rclone
+      uses: AnimMouse/setup-rclone@v1
+
+    - name: Create diagnostics and upload to Google Cloud Storage
       env:
         service_account_json: ${{ secrets.GCS_SERVICE_ACCOUNT_1 }}
       run: |
-        mkdir -p $HOME/.cache
-        pushd $HOME/.cache
-        curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip
-        popd
-        unzip $HOME/.cache/rclone-current-linux-amd64.zip
-        ls -d rclone-v* > $GITHUB_PATH
         echo "$service_account_json" >ci/service-account-key.json
-
-    - name: Create diagnostics and upload to Google Cloud Storage
-      if: env.pr_from_main_repo && matrix.run-diagnostics
-      run: |
         item historical diagnostics output/
         rclone --progress copy output ${{ env.gcs_bucket }}${{ github.run_id }}/
 
     - uses: LouisBrunner/[email protected]
-      if: env.pr_from_main_repo && matrix.run-diagnostics
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         name: Upload historical database & diagnostics
         conclusion: success
         details_url: ${{ env.gcs_url }}${{ github.run_id }}/index.html
         output: |
           {"summary": "${{ env.gcs_url }}${{ github.run_id }}/index.html"}
+
+  pre-commit:
+    name: Code quality
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+
+    - name: Force recreation of pre-commit virtual environment for mypy
+      if: github.event_name == 'schedule'  # Comment this line to run on a PR
+      run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true
+      env: { GH_TOKEN: "${{ github.token }}" }
+
+    - uses: pre-commit/[email protected]

.gitignore

@@ -15,10 +15,12 @@ ipython_config.py
 *.log
 *.pdf
 
+.benchmarks
 .cache
 .coverage*
 .mypy_cache
 .pytest_cache
+.ruff_cache
 __pycache__
 build
 coverage.xml

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.7.0
+  hooks:
+  - id: mypy
+    additional_dependencies:
+    - click
+    - iam-units
+    - Jinja2
+    - lxml-stubs
+    - pandas-stubs
+    - Pint
+    - pytest
+    - sdmx1
+    - traitlets
+    - types-PyYAML
+    - types-python-dateutil
+    - types-requests
+    - types-setuptools
+    - xarray
+    args: []
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.1.2
+  hooks:
+  - id: ruff
+  - id: ruff-format
+    args: [ --check ]

item/historical/T003.py

@@ -89,8 +89,8 @@ def lookup(value):
 
     df = pd.concat([df, df["Variable"].apply(lookup)], axis=1)
 
-    return (
-        # Compute partial sums that exclude pipelines
+    # Compute partial sums that exclude pipelines
+    df0 = (
         # Select only the subset of variables, then group by Country and TIME_PERIOD
         df[df["Variable"].isin(PARTIAL)]
         .groupby(["Country", "TIME_PERIOD"])
@@ -101,8 +101,13 @@ def lookup(value):
         .reset_index()
         # Assign other dimensions for this sum
         .assign(mode="Inland ex. pipeline")
-        # Concatenate with the original data
-        .append(df, ignore_index=True)
+    )
+
+    # - Concatenate with the original data.
+    # - Fill "operator" and "vehicle" key values.
+    # - Sort.
+    return (
+        pd.concat([df, df0], ignore_index=True)
         .fillna({"operator": "_T", "vehicle": "_T"})
         .sort_values(by=["Country", "TIME_PERIOD", "mode", "vehicle"])
     )

item/historical/T012.py

@@ -1,6 +1,4 @@
 """Data cleaning code and configuration for T012."""
-import numpy as np
-
 from item.util import convert_units, dropna_logged
 
 #: iTEM data flow matching the data from this source.
@@ -54,7 +52,7 @@ def process(df):
             Value=lambda df_: df_["Value"]
             .str.replace(" ", "")
             .replace("...", "NaN")
-            .astype(np.float)
+            .astype(float)
         )
         .pipe(dropna_logged, "Value", [COLUMNS["country_name"]])
         .pipe(convert_units, "kpassenger", "Mpassenger")