Merge branch 'nexus-upd' of https://github.com/iiasa/message-ix-models …

…into nexus-upd

.github/codecov.yml

@@ -1,3 +1,13 @@
 comment:
   layout: "diff, files"
   behavior: once
+
+coverage:
+  precision: 1
+  status:
+    project:
+      default:
+        if_ci_failed: success
+    patch:
+      default:
+        if_ci_failed: success

doc/api/disutility.rst

@@ -1,7 +1,7 @@
 .. currentmodule:: message_ix_models.model.disutility
 
-Consumer disutility
-*******************
+Consumer disutility (:mod:`model.disutility`)
+*********************************************
 
 This module provides a generalized consumer disutility formulation, currently used by :mod:`message_data.model.transport`.
 The formulation rests on the concept of “consumer groups”; each consumer group may have a distinct disutility associated with using the outputs of each technology.

doc/api/model-snapshot.rst

@@ -1,7 +1,7 @@
 .. currentmodule:: message_ix_models.model.snapshot
 
-Load MESSAGEix-GLOBIOM snapshots (:mod:`.model.snapshot`)
-*********************************************************
+Load model snapshots (:mod:`.model.snapshot`)
+*********************************************
 
 This code allows to fetch *snapshots* containing completely parametrized MESSAGEix-GLOBIOM model instances, and load these into :class:`Scenarios <message_ix.Scenario>`.
 

doc/api/model.rst

@@ -1,11 +1,33 @@
+.. currentmodule:: message_ix_models.model
+
 Models and variants (:mod:`~message_ix_models.model`)
 *****************************************************
 
-.. currentmodule:: message_ix_models.model
+Submodules described on this page:
+
+.. contents::
+   :local:
+   :backlinks: none
+
+Submodules described on separate pages:
+
+- :doc:`/api/model-bare`
+- :doc:`/api/model-build`
+- :doc:`/api/disutility`
+- :doc:`/api/model-emissions`
+- :doc:`/api/model-snapshot`
 
 .. automodule:: message_ix_models.model
    :members:
 
+.. currentmodule:: message_ix_models.model.macro
+
+:mod:`.model.macro`: MESSAGE-MACRO
+==================================
+
+.. automodule:: message_ix_models.model.macro
+   :members:
+
 :mod:`.model.structure`: Model structure information
 ====================================================
 

doc/api/tools.rst

@@ -18,8 +18,8 @@ On this page:
    :members:
 
 
-ADVANCE data
-============
+ADVANCE data (:mod:`.tools.advance`)
+====================================
 
 .. currentmodule:: message_ix_models.tools.advance
 

doc/index.rst

@@ -34,7 +34,6 @@ Among other tasks, the tools allow modelers to:
    api/model-emissions
    api/model-snapshot
    api/disutility
-   api/project
    api/tools
    api/util
    api/testing
@@ -45,6 +44,7 @@ Among other tasks, the tools allow modelers to:
    :caption: Variants and projects
 
    water/index
+   api/project
 
 .. toctree::
    :maxdepth: 2

doc/whatsnew.rst

@@ -1,11 +1,18 @@
 What's new
 **********
 
-Next release
-============
+.. Next release
+.. ============
+
+v2023.7.26
+==========
 
 - Add code and CLI commands to :doc:`fetch and load MESSAGEix-GLOBIOM snapshots <api/model-snapshot>` (:pull:`102`).
 - Add :func:`.util.pooch.fetch`, a thin wrapper for using :doc:`Pooch <pooch:about>` (:pull:`102`).
+- New module :mod:`message_ix_models.model.macro` with utilities for calibrating :mod:`message_ix.macro` (:pull:`104`).
+- New method :meth:`.Workflow.guess_target` (:pull:`104`).
+- Change in behaviour of :meth:`.Workflow.add_step`: the method now returns the name of the newly-added workflow step, rather than the :class:`WorkflowStep` object added to carry out the step (:pull:`104`).
+  The former is more frequently used in code that uses :class:`.Workflow`.
 
 v2023.5.31
 ==========

message_ix_models/data/test/macro/kgdp.csv

@@ -0,0 +1,16 @@
+# Converted from P:/ene.model/MACRO/python/R12-CHN-5y_macro_data_NGFS_w_rc_ind_adj_mat.xlsx
+#
+# Units: dimensionless
+node,value
+R12_AFR,3.0
+R12_RCPA,3.0
+R12_EEU,3.0
+R12_FSU,3.0
+R12_LAM,3.0
+R12_MEA,3.0
+R12_NAM,2.4
+R12_PAO,2.8
+R12_PAS,3.0
+R12_SAS,3.0
+R12_WEU,2.8
+R12_CHN,3.0

message_ix_models/model/config.py

@@ -6,7 +6,22 @@
 
 @dataclass
 class Config:
-    """Settings and valid values for :mod:`message_ix_models.model` and submodules."""
+    """Settings and valid values for :mod:`message_ix_models.model` and submodules.
+
+    For backwards compatibility, it is possible to access these on a :class:`Context`
+    using:
+
+    .. code-block:: python
+
+       c = Context()
+       c.regions = "R14"
+
+    …however, it is best to access them explicitly as:
+
+    .. code-block:: python
+
+       c.model.regions = "R14"
+    """
 
     #: The 'node' codelist (regional aggregation) to use. Must be one of the lists of
     #: nodes described at :doc:`/pkg-data/node`.

message_ix_models/model/macro.py

@@ -0,0 +1,163 @@
+"""Tools for calibrating MACRO for MESSAGEix-GLOBIOM.
+
+See :doc:`message-ix:macro` for *general* documentation on MACRO and MESSAGE-MACRO. This
+module contains tools specifically for using these models with MESSAGEix-GLOBIOM.
+"""
+import logging
+from functools import lru_cache
+from itertools import product
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Literal, Mapping, Optional, Union
+
+import pandas as pd
+
+from message_ix_models.model.bare import get_spec
+from message_ix_models.util import nodes_ex_world
+
+if TYPE_CHECKING:  # pragma: no cover
+    from sdmx.model.v21 import Code
+
+    from message_ix_models import Context
+
+log = logging.getLogger(__name__)
+
+#: Default set of commodities to include in :func:`generate`.
+COMMODITY = ["i_therm", "i_spec", "rc_spec", "rc_therm", "transport"]
+
+
+def generate(
+    parameter: Literal["aeei", "config", "depr", "drate", "lotol"],
+    context: "Context",
+    commodities: Union[List[str], List["Code"]] = COMMODITY,
+    value: Optional[float] = None,
+) -> pd.DataFrame:
+    """Generate uniform data for one :mod:`message_ix.macro` `parameter`.
+
+    :meth:`message_ix.Scenario.add_macro` expects as its `data` parameter a
+    :class:`dict` that maps certain MACRO parameter names (or the special name "config")
+    to :class:`.pandas.DataFrame`. This function generates data for those data frames.
+
+    For the particular dimensions, generate automatically includes:
+
+    - "node": All nodes in the node code list given by :func:`.nodes_ex_world`, for the
+      node list indicated by :attr:`.model.Config.regions`.
+    - "year": All periods from the period *before* the first model year.
+    - "commodity": The elements of `commodities`.
+    - "sector": If each entry of `commodities` is a :class:`.Code` and has an annotation
+      with id="macro-sector", the value of that annotation. Otherwise, the same as
+      `commodity`.
+
+    `value` supplies the parameter value, which is the same for all observations.
+    The labels level="useful" and unit="-" are fixed.
+
+    Parameters
+    ----------
+    parameter : str
+        MACRO parameter for which to generate data.
+    context
+        Used with :func:`.bare.get_spec`.
+    commodities : list of str or Code
+        Commodities to include in the MESSAGE-MACRO linkage.
+    value : float
+        Parameter value.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The columns vary according to `parameter`:
+
+        - "aeei": node, sector, year, value, unit.
+        - "depr", "drate", or "lotol": node, value, unit.
+        - "config": node, sector, commodity, level, year.
+    """
+    spec = get_spec(context)
+
+    if isinstance(commodities[0], str):
+        c_codes = spec.add.set["commodity"]
+    else:
+        c_codes = commodities
+
+    @lru_cache
+    def _sector(commodity: str) -> str:
+        try:
+            idx = c_codes.index(commodity)
+            return str(c_codes[idx].get_annotation(id="macro-sector").text)
+        except (KeyError, ValueError) as e:
+            log.info(e)
+            return str(commodity)
+
+    # AEEI data must begin from the period before the first model period
+    y0_index = spec.add.set["year"].index(spec.add.y0)
+    iterables = dict(
+        c_s=zip(  # Paired commodity and sector
+            map(str, commodities), map(_sector, commodities)
+        ),
+        level=["useful"],
+        node=nodes_ex_world(spec.add.N),
+        sector=map(_sector, commodities),
+        year=spec.add.set["year"][y0_index:],
+    )
+
+    if parameter == "aeei":
+        dims = ["node", "year", "sector"]
+        iterables.update(year=spec.add.set["year"][y0_index - 1 :])
+    elif parameter == "config":
+        dims = ["node", "c_s", "level", "year"]
+        assert value is None
+    elif parameter in ("depr", "drate", "lotol"):
+        dims = ["node"]
+    else:
+        raise NotImplementedError(f"generate(…) for MACRO parameter {parameter!r}")
+
+    result = pd.DataFrame(
+        [tuple(values) for values in product(*[iterables[d] for d in dims])],
+        columns=dims,
+    )
+
+    if parameter == "config":
+        return pd.concat(
+            [
+                result.drop("c_s", axis=1),
+                pd.DataFrame(result["c_s"].tolist(), columns=["commodity", "sector"]),
+            ],
+            axis=1,
+        )
+    else:
+        return result.assign(value=value, unit="-")
+
+
+def load(base_path: Path) -> Mapping[str, pd.DataFrame]:
+    """Load MACRO data from CSV files.
+
+    The function reads files in the simple/long CSV format understood by
+    :func:`genno.computations.load_file`. For use with
+    :meth:`~message_ix.Scenario.add_macro`, the dimension names should be given in full,
+    for instance "node" or "sector".
+
+    Parameters
+    ----------
+    base_path : pathlib.Path
+        Directory containing zero or more CSV files.
+
+    Returns
+    -------
+    dict of (str -> pandas.DataFrame)
+        Mapping from MACRO calibration parameter names to data; one entry for each file
+        in `base_path`.
+    """
+    from genno.computations import load_file
+
+    result = {}
+    for filename in base_path.glob("*.csv"):
+        name = filename.stem
+
+        q = load_file(filename, name=name)
+
+        result[name] = (
+            q.to_frame()
+            .reset_index()
+            .rename(columns={name: "value"})
+            .assign(unit=f"{q.units:~}" or "-")
+        )
+
+    return result

message_ix_models/tests/model/test_macro.py

@@ -0,0 +1,48 @@
+import pandas as pd
+import pandas.testing as pdt
+import pytest
+from sdmx.model import Annotation, Code
+
+from message_ix_models.model.macro import generate, load
+from message_ix_models.util import package_data_path
+
+
+@pytest.mark.parametrize(
+    "parameter, value",
+    [
+        ("aeei", 1.0),
+        ("config", None),
+        ("depr", 1.0),
+        ("drate", 1.0),
+        ("lotol", 1.0),
+        pytest.param("foo", 1.0, marks=pytest.mark.xfail(raises=NotImplementedError)),
+    ],
+)
+def test_generate0(test_context, parameter, value):
+    result = generate(parameter, test_context, value=value)
+
+    assert not result.isna().any(axis=None)
+
+
+def test_generate1(test_context):
+    commodities = [
+        Code(id="foo", annotations=[Annotation(id="macro-sector", text="BAR")]),
+        Code(id="baz", annotations=[Annotation(id="macro-sector", text="QUX")]),
+    ]
+
+    result = generate("config", test_context, commodities)
+
+    assert {"foo", "baz"} == set(result["commodity"].unique())
+
+    # Only the identified sectors appear
+    assert {"BAR", "QUX"} == set(result["sector"].unique())
+
+    # Only 2 unique (commodity, sector) combinations appear
+    assert 2 == len(result[["commodity", "sector"]].drop_duplicates())
+
+
+def test_load(test_context):
+    result = load(package_data_path("test", "macro"))
+    assert {"kgdp"} == set(result.keys())
+    pdt.assert_index_equal(pd.Index(["node", "value", "unit"]), result["kgdp"].columns)
+    assert not result["kgdp"].isna().any(axis=None)