Merge pull request #225 from iiasa/transport/2024-W36

Transport improvements for 2024-W36

.github/workflows/transport.yaml

@@ -1,7 +1,10 @@
 name: MESSAGEix-Transport
 
 env:
-  target: transport.yaml --repo=iiasa/message_data
+  # The repository, ref (branch), and workflow file name to dispatch
+  target-repo: iiasa/message_data
+  target-ref: dev
+  target-workflow: transport
 
   # Starting point of the workflow.
   # Use this value to build from a certain scenario:
@@ -33,12 +36,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Invoke "transport.yaml" workflow in message_data
+      - name: Invoke "${{ env.target-workflow }}" workflow in ${{ env.target-repo }}
         env:
           GH_TOKEN: ${{ secrets.MESSAGE_DATA_DISPATCH_TOKEN }}
         run: |
           gh workflow run \
-            ${{ env.target }} --ref=dev \
+            ${{ env.target-workflow }}.yaml \
+            --repo=${{ env.target-repo }} \
+            --ref=${{ env.target-ref }} \
             --field ref=${{ github.ref }} \
             --field sha=${{ github.sha }} \
             --field base=${{ env.base }} \
@@ -47,6 +52,6 @@ jobs:
           sleep 5
 
           gh run list \
-            --workflow=${{ env.target }} \
+            --workflow=${{ env.target-workflow.yaml }} --repo=${{ env.target-repo }} \
             --json url,status \
             --jq 'map(select(.status != "completed"))[0].url' >> $GITHUB_STEP_SUMMARY

.readthedocs.yaml

@@ -10,13 +10,16 @@ formats:
 
 # Set the version of Python and other tools you might need
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
+    python: "3.13"
   jobs:
     post_install:
     # Remove requests-cache, which appears to cause segfaults on RTD workers
     - pip uninstall --yes requests-cache
+    # TEMPORARY With Python 3.13 pyam-iamc resolves to 1.3.1, which in turn
+    # limits pint < 0.17. Override.
+    - pip install --upgrade pint
 
 
 # Build documentation in the docs/ directory with Sphinx
@@ -28,5 +31,4 @@ python:
   - requirements: doc/requirements.txt
   - method: pip
     path: .
-    extra_requirements:
-      - docs
+    extra_requirements: [ docs ]

doc/api/util.rst

@@ -96,15 +96,18 @@ Commonly used:
    :members:
    :exclude-members: clone_to_dest
 
-   Context is a subclass of :class:`dict`, so common methods like :meth:`~dict.copy` and :meth:`~dict.setdefault` may be used to handle settings.
-   To be forgiving, it also provides attribute access; ``context.foo`` is equivalent to ``context["foo"]``.
+   Context is :class:`dict`-like, so :class:`dict` methods including :meth:`~dict.copy` and :meth:`~dict.setdefault` may be used to handle settings.
+   It also provides attribute access: :py:`context.foo` is equivalent to :py:`context["foo"]`.
 
    A Context instance always has the following members:
 
-   - ``core``: an instance of :class:`message_ix_models.Config`.
-   - ``model``: an instance of :class:`message_ix_models.model.Config`.
+   1. :attr:`core`: an instance of :class:`message_ix_models.Config`.
+   2. :attr:`model`: an instance of :class:`message_ix_models.model.Config`.
+   3. :attr:`report`: an instance of :class:`message_ix_models.report.Config`.
 
-   Attributes of these classes may be accessed by shorthand, e.g. ``context.regions`` is shorthand for ``context.model.regions``.
+   Attributes of (1) and (2) **may** be accessed by shorthand/aliases.
+   For instance, :py:`context.regions` is an alias for :py:`context.model.regions`.
+   However, for clarity and to support type checking, explicit reference to the configuration class and its attributes **should** be used.
 
    Context provides additional methods to do common tasks that depend on configurable settings:
 
@@ -212,3 +215,11 @@ Commonly used:
 
 .. automodule:: message_ix_models.util.sdmx
    :members:
+
+:mod:`.types`
+=============
+
+.. currentmodule:: message_ix_models.types
+
+.. automodule:: message_ix_models.types
+   :members:

doc/pkg-data/codelists.rst

@@ -25,6 +25,14 @@ These codes have the following annotations:
 .. literalinclude:: ../../message_ix_models/data/commodity.yaml
    :language: yaml
 
+.. _emission-yaml:
+
+Emission species (:file:`emission.yaml`)
+========================================
+
+.. literalinclude:: ../../message_ix_models/data/emission.yaml
+   :language: yaml
+
 .. _level-yaml:
 
 Levels (``level.yaml``)

doc/project/ssp.rst

@@ -41,3 +41,16 @@ Data
    .. autosummary::
       SSPOriginal
       SSPUpdate
+
+2024 update
+===========
+
+Transport
+---------
+
+.. currentmodule:: message_ix_models.project.ssp.transport
+
+.. automodule:: message_ix_models.project.ssp.transport
+   :members:
+
+   Use :program:`mix-models ssp transport --help in.xlsx out.xlsx` to invoke :func:`.main`.

doc/requirements.in

@@ -1,6 +1,6 @@
-# Input file for pip-compile 
+# Input file for pip-compile
 # Only specify the packages necessary for [docs]
 
-sphinx==6.2.1
-sphinx-rtd-theme==1.2.2
-sphinxcontrib-bibtex==2.5.0
+sphinx==8.1.3
+sphinx-rtd-theme==3.0.2
+sphinxcontrib-bibtex==2.6.3

doc/requirements.txt

@@ -1,18 +1,18 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile --allow-unsafe --output-file=requirements.txt requirements.in
 #
-alabaster==0.7.13
+alabaster==1.0.0
     # via sphinx
-babel==2.12.1
+babel==2.16.0
     # via sphinx
 certifi==2023.7.22
     # via requests
 charset-normalizer==3.2.0
     # via requests
-docutils==0.18.1
+docutils==0.21.2
     # via
     #   pybtex-docutils
     #   sphinx
@@ -36,7 +36,7 @@ pybtex==0.24.0
     #   sphinxcontrib-bibtex
 pybtex-docutils==1.0.2
     # via sphinxcontrib-bibtex
-pygments==2.16.1
+pygments==2.18.0
     # via sphinx
 pyyaml==6.0.1
     # via pybtex
@@ -48,29 +48,29 @@ six==1.16.0
     #   pybtex
 snowballstemmer==2.2.0
     # via sphinx
-sphinx==6.2.1
+sphinx==8.1.3
     # via
     #   -r requirements.in
     #   sphinx-rtd-theme
     #   sphinxcontrib-bibtex
     #   sphinxcontrib-jquery
-sphinx-rtd-theme==1.2.2
+sphinx-rtd-theme==3.0.2
     # via -r requirements.in
-sphinxcontrib-applehelp==1.0.4
+sphinxcontrib-applehelp==2.0.0
     # via sphinx
-sphinxcontrib-bibtex==2.5.0
+sphinxcontrib-bibtex==2.6.3
     # via -r requirements.in
-sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-devhelp==2.0.0
     # via sphinx
-sphinxcontrib-htmlhelp==2.0.1
+sphinxcontrib-htmlhelp==2.1.0
     # via sphinx
 sphinxcontrib-jquery==4.1
     # via sphinx-rtd-theme
 sphinxcontrib-jsmath==1.0.1
     # via sphinx
-sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-qthelp==2.0.0
     # via sphinx
-sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-serializinghtml==2.0.0
     # via sphinx
 urllib3==2.0.4
     # via requests

doc/transport/input.rst

@@ -159,6 +159,24 @@ node = R12_SAS [2]_
 .. [2] A. Javaid, `message_data#180 (comment) <https://github.com/iiasa/message_data/issues/180#issuecomment-1941860412>`_.
 .. [3] A. Javaid, `message_data#538 (comment) <https://github.com/iiasa/message_data/issues/538#issuecomment-1934663340>`__.
 
+:file:`pdt-cap.csv` → ``P activity:scenario-n-t-y:exo``
+-------------------------------------------------------
+
+Measure
+   Projected PDT per capita
+Dimensions
+   :math:`(s, n, t, y)`
+Units:
+   km / passenger / year
+
+- This file is only used for :math:`s` values such as :py:`scenario="LED"`, in which case it is the source for projected
+PDT per capita.
+- Values for :py:`scenario="LED"` added in :pull:`225`.
+  Method described in `this Slack message <https://iiasa-ece.slack.com/archives/CCFHDNA6P/p1731510626983289?thread_ts=1730218237.960269&cid=CCFHDNA6P>`__.
+
+  .. todo:: Transcribe the method into this document.
+
+
 :file:`pdt-cap-ref.csv` → ``pdt:n:capita+ref``
 ----------------------------------------------
 
@@ -197,6 +215,15 @@ Units
 Where/how used
    :func:`.pdt_per_capita`.
 
+:file:`load-factor-ldv.csv` → ``load factor ldv:scenario-n-y:exo``
+------------------------------------------------------------------
+
+- Original source: Duplicate of :file:`…/R11/load-factor-ldv.csv` with R12_CHN and R12_RCPA values filled from R11_CPA.
+- Values for :py:`scenario="LED"` added in :pull:`225`.
+  Method described in `this Slack message <https://iiasa-ece.slack.com/archives/CCFHDNA6P/p1731914351904059?thread_ts=1730218237.960269&cid=CCFHDNA6P>`_.
+
+  .. todo:: Transcribe the method into this document.
+
 Other files
 -----------
 - :file:`demand-scale.csv` → ``demand scale:n-y:exo``

doc/whatsnew.rst

@@ -4,30 +4,103 @@ What's new
 Next release
 ============
 
-- Introduce an SSP configuration for cooling technologies with share constraints (:pull:`256`).
 - :mod:`message_ix_models` is tested and compatible with `Python 3.13 <https://www.python.org/downloads/release/python-3130/>`__ (:pull:`250`).
 - Support for Python 3.8 is dropped (:pull:`250`), as it has reached end-of-life.
-- Connect the water module to the cost module for cooling technologies (:pull:`245`).
-- Make setup of constraints for cooling technologies flexible and update solar csp tech. name (:pull:`242`).
-- Fix the nexus/cooling function and add test for checking some input data (:pull:`236`).
-- Add :doc:`/project/circeular` project code and documentation (:pull:`232`).
-- Update water availability data and major code editing to allow a new test suite for the water module (:pull:`106`).
 - Expand :doc:`repro` with sections on :ref:`repro-doc` and :ref:`versioning`, including :ref:`a list of external model names and ‘versions’ <model-names>` like “MESSAGEix-GLOBIOM 2.0” (:issue:`224`, :pull:`226`).
-- Update :doc:`/transport/index` (:pull:`213`).
-- Add "LED", "SSP4", and "SSP5" as values for the :program:`--ssp=…` option in :func:`.common_params` (:pull:`233`).
-- Fix and update :doc:`/api/tools-costs` (:pull:`219`, :pull:`206`, :pull:`221`, :pull:`227`, :pull:`222`, :pull:`255`)
-
-  - Fix naming of GDP and population columns in SSP data aggregation (:pull:`219`).
-  - Edit inputs for storage, CSP, hydrogen, and industry technologies (:pull:`206`).
-  - Replace solar and wind technologies with new ones (:pull:`206`).
-  - Reorganize input files and incorporate `first_year.csv` data into `tech_map.csv` (:pull:`221`).
-  - Reconfigure use and implementation of technology variants/modules to be more agnostic (:pull:`221`).
-  - Change cost decay to reach reduction percentage specified on the year 2100 (:pull:`227`).
-  - Add `cooling` technology variant/module (:pull:`222`).
-  - Add functionality to specify cost reduction values and cost reduction scenarios in a module (:pull:`255`).
+- New code list :ref:`emission-yaml` (:pull:`225`).
+
+By topic:
+
+.. contents::
+   :local:
+   :backlinks: none
+
+Materials
+---------
+
 - Improve and extend :doc:`/material/index` (:pull:`218`, :pull:`253`).
+- Release of MESSAGEix-Materials 1.1.0 (:doc:`/material/v1.1.0`).
+
+Transport
+---------
 
-  - Release of MESSAGEix-Materials 1.1.0 (:doc:`/material/v1.1.0`).
+- Update :doc:`/transport/index` (:pull:`213`, :pull:`225`).
+- Rework :mod:`~.transport.freight`, :mod:`~.transport.ldv`, and :mod:`~.transport.non_ldv` to use :mod:`genno` consistently.
+- Adopt consistent terms "F RAIL" and "F ROAD" for freight service/modes.
+- New technologies: "f rail {electr,lightoil}", "f road electr".
+- Extend unit annotations to all transport commodities.
+- New input :doc:`files </transport/input>` :file:`pdt-cap.csv` and :file:`load-factor-ldv.csv`.
+- Add :program:`mix-models ssp transport` CLI command to postprocess aviation emissions data.
+- New SDMX Codelist ``IIASA:CL_TRANSPORT_SCENARIO`` to distinguish “Low energy demand (LED)” and :doc:`/project/edits` scenarios from :doc:`/project/ssp` baseline scenarios.
+- Adjust R12 baseline settings:
+
+  - :file:`mode-share/default.csv`: adjust ``R12_NAM`` values for AIR and LDV.
+  - :file:`pdt-cap-ref.csv`: adjust ``R12_NAM`` value.
+- Implement LED scenarios via :file:`load-factor-ldv.csv` and :file:`pdt-cap.csv`.
+- Use y=2019 data from IEA EWEB 2024 edition to align MESSAGEix-Transport with base model calibration.
+- New :mod:`genno` operators: :func:`~.transport.operator.broadcast`,  :func:`~.transport.operator.broadcast_wildcard`,  :func:`~.transport.operator.broadcast_t_c_l`, :func:`~.transport.operator.freight_usage_output`.
+- New utility methods :func:`~.transport.config.get_cl_scenario`, :func:`~.transport.util.wildcard`.
+- Replace :class:`DataSourceConfig.LDV <.transport.config.DataSourceConfig>` setting with :attr:`Config.dummy_LDV <.transport.config.Config.dummy_LDV>`.
+
+Water/Nexus
+-----------
+
+- Update water availability data and major code editing to allow a new test suite for doc:`/water/index` (:pull:`106`).
+- Fix the nexus/cooling function and add test for checking some input data (:pull:`236`).
+- Make setup of constraints for cooling technologies flexible and update solar CSP technology name (:pull:`242`).
+- Connect :mod:`.model.water` to :mod:`.tools.costs` for cooling technologies (:pull:`245`).
+- Introduce an SSP configuration for cooling technologies with share constraints (:pull:`256`).
+
+Investment and fixed costs
+--------------------------
+
+- Fix naming of GDP and population columns in SSP data aggregation for :doc:`/api/tools-costs` (:pull:`219`).
+- Edit inputs for storage, CSP, hydrogen, and industry technologies (:pull:`206`).
+- Replace solar and wind technologies with new ones (:pull:`206`).
+- Reorganize input files and incorporate `first_year.csv` data into `tech_map.csv` (:pull:`221`).
+- Reconfigure use and implementation of technology variants/modules to be more agnostic (:pull:`221`).
+- Change cost decay to reach reduction percentage specified on the year 2100 (:pull:`227`).
+- Add `cooling` technology variant/module (:pull:`222`).
+- Add functionality to specify cost reduction values and cost reduction scenarios in a module (:issue:`251`, :pull:`255`).
+
+Others
+------
+
+- Add "LED", "SSP4", and "SSP5" as values for the :program:`--ssp=…` option in :func:`.common_params` (:pull:`233`).
+- Add :doc:`/project/circeular` project code and documentation (:pull:`232`).
+- :mod:`.iea.web` handles the 2024 edition and fixed-width file format published by the IEA directly (:pull:`225`).
+
+  - Code lists for the ``COUNTRY``, ``FLOW``, and ``PRODUCT`` concepts are included with :mod:`message_ix_models`.
+  - Add :data:`~.iea.web.COUNTRY_NAME` to map particular labels appearing in these data.
+  - Fuzzed data and tests for this functionality.
+
+- Add :any:`.types.ParameterData` and :any:`.types.MutableParameterData` to type the common internal data structure in which a :class:`dict` maps from MESSAGE parameter names to :mod:`message_ix`-structured :class:`pandas.DataFrame` (:pull:`225`).
+- :class:`message_ix_models.Config`—the “core” configuration class—gains methods specific to its settings (:pull:`225`).
+  These were formerly on :class:`.Context`.
+- :class:`.Context` is no longer a subclass of :class:`dict` (:pull:`225`).
+  This avoids its mishandling by :mod:`dask` version 2024.11.0 or later when Context is used with :class:`ixmp.Reporter`.
+
+  - Add :data:`.MODULE_WITH_CONFIG_DATACLASS`;
+    add attributes such as :attr:`.Context.model` that are typed to the respective class such as :class:`.model.Config`.
+  - Add :meth:`.Context.asdict` for serialization.
+  - Former methods such as :meth:`.Context.get_scenario` are aliased to their new locations, e.g. :meth:`.Config.get_scenario`.
+
+- Improve :class:`.ScenarioInfo` (:pull:`225`):
+
+  - Implement the :py:`|` (logical OR/union) operator: `si_a | si_b` is a new ScenarioInfo instance with the union of the contents of the operands.
+  - New method :meth:`.substitute_codes` to replace string codes (for instance, determined from a Scenario object) with Codes from the corresponding code list, including all annotations.
+
+- :class:`.MappingAdapter` skips missing labels in the input data without raising an exception (:pull:`225`).
+- :meth:`.Workflow.visualize` displays in left-to-right rank direction by default (:pull:`225`).
+- :func:`.convert_units` can handle MESSAGE-scheme :class:`pandas.DataFrame` (:pull:`225`).
+- :func:`.util.sdmx.make_enum` uses :class:`.URNLookupEnum` by default (:pull:`225`).
+  This allows to call, for instance, :py:`SSP_2024.by_urn("…Code=ICONICS:SSP(2024).2")` to retrieve an enumeration number.
+- :func:`.make_matched_dfs` accepts :class:`dict` for its :py:`par_value` arg, allowing replacement of values for particular dimensions of output data frames (:pull:`225`).
+- New reporting operator :func:`.quantity_from_iamc` (:pull:`225`).
+- :func:`.same_node` and :func:`.same_time` can handle :any:`.ParameterData` (:pull:`225`).
+- :func:`.report.register` becomes :meth:`.report.Config.register` which populates :attr:`.report.Config.callback` (:pull:`225`).
+  The set of callback functions used to configure a class:`.Reporter` instance is thus now particular to a :class:`.report.Config` instance attached to a :class:`.Context` instance.
+  This allows better isolation of code/tests that use different sets of callbacks.
 
 v2024.8.6
 =========

message_ix_models/cli.py

@@ -164,7 +164,7 @@ def _log_threads(k: int, n: int):
     "message_ix_models.model.water.cli",
     "message_ix_models.project.circeular.cli",
     "message_ix_models.project.edits.cli",
-    "message_ix_models.project.ssp",
+    "message_ix_models.project.ssp.cli",
     "message_ix_models.report.cli",
     "message_ix_models.model.material.cli",
     "message_ix_models.testing.cli",