diff --git a/.github/codecov.yml b/.github/codecov.yml index 01fa01a41c..f6db6cba29 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -1,3 +1,13 @@ comment: layout: "diff, files" behavior: once + +coverage: + precision: 1 + status: + project: + default: + if_ci_failed: success + patch: + default: + if_ci_failed: success diff --git a/doc/api/disutility.rst b/doc/api/disutility.rst index 088e8216aa..67de6811d4 100644 --- a/doc/api/disutility.rst +++ b/doc/api/disutility.rst @@ -1,7 +1,7 @@ .. currentmodule:: message_ix_models.model.disutility -Consumer disutility -******************* +Consumer disutility (:mod:`model.disutility`) +********************************************* This module provides a generalized consumer disutility formulation, currently used by :mod:`message_data.model.transport`. The formulation rests on the concept of “consumer groups”; each consumer group may have a distinct disutility associated with using the outputs of each technology. diff --git a/doc/api/model-snapshot.rst b/doc/api/model-snapshot.rst index 9efcb4ee54..e4ba2fc083 100644 --- a/doc/api/model-snapshot.rst +++ b/doc/api/model-snapshot.rst @@ -1,7 +1,7 @@ .. currentmodule:: message_ix_models.model.snapshot -Load MESSAGEix-GLOBIOM snapshots (:mod:`.model.snapshot`) -********************************************************* +Load model snapshots (:mod:`.model.snapshot`) +********************************************* This code allows to fetch *snapshots* containing completely parametrized MESSAGEix-GLOBIOM model instances, and load these into :class:`Scenarios `. diff --git a/doc/api/model.rst b/doc/api/model.rst index 28f64655bd..74950f8742 100644 --- a/doc/api/model.rst +++ b/doc/api/model.rst @@ -1,11 +1,33 @@ +.. currentmodule:: message_ix_models.model + Models and variants (:mod:`~message_ix_models.model`) ***************************************************** -.. currentmodule:: message_ix_models.model +Submodules described on this page: + +.. contents:: + :local: + :backlinks: none + +Submodules described on separate pages: + +- :doc:`/api/model-bare` +- :doc:`/api/model-build` +- :doc:`/api/disutility` +- :doc:`/api/model-emissions` +- :doc:`/api/model-snapshot` .. automodule:: message_ix_models.model :members: +.. currentmodule:: message_ix_models.model.macro + +:mod:`.model.macro`: MESSAGE-MACRO +================================== + +.. automodule:: message_ix_models.model.macro + :members: + :mod:`.model.structure`: Model structure information ==================================================== diff --git a/doc/api/tools.rst b/doc/api/tools.rst index 2057c07924..312a2bb45d 100644 --- a/doc/api/tools.rst +++ b/doc/api/tools.rst @@ -18,8 +18,8 @@ On this page: :members: -ADVANCE data -============ +ADVANCE data (:mod:`.tools.advance`) +==================================== .. currentmodule:: message_ix_models.tools.advance diff --git a/doc/index.rst b/doc/index.rst index 58b23ddeb1..5c61e68894 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -34,7 +34,6 @@ Among other tasks, the tools allow modelers to: api/model-emissions api/model-snapshot api/disutility - api/project api/tools api/util api/testing @@ -45,6 +44,7 @@ Among other tasks, the tools allow modelers to: :caption: Variants and projects water/index + api/project .. toctree:: :maxdepth: 2 diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 88af26e3b8..6d6d736236 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -1,11 +1,18 @@ What's new ********** -Next release -============ +.. Next release +.. ============ + +v2023.7.26 +========== - Add code and CLI commands to :doc:`fetch and load MESSAGEix-GLOBIOM snapshots ` (:pull:`102`). - Add :func:`.util.pooch.fetch`, a thin wrapper for using :doc:`Pooch ` (:pull:`102`). +- New module :mod:`message_ix_models.model.macro` with utilities for calibrating :mod:`message_ix.macro` (:pull:`104`). +- New method :meth:`.Workflow.guess_target` (:pull:`104`). +- Change in behaviour of :meth:`.Workflow.add_step`: the method now returns the name of the newly-added workflow step, rather than the :class:`WorkflowStep` object added to carry out the step (:pull:`104`). + The former is more frequently used in code that uses :class:`.Workflow`. v2023.5.31 ========== diff --git a/message_ix_models/data/test/macro/kgdp.csv b/message_ix_models/data/test/macro/kgdp.csv new file mode 100644 index 0000000000..501c982452 --- /dev/null +++ b/message_ix_models/data/test/macro/kgdp.csv @@ -0,0 +1,16 @@ +# Converted from P:/ene.model/MACRO/python/R12-CHN-5y_macro_data_NGFS_w_rc_ind_adj_mat.xlsx +# +# Units: dimensionless +node,value +R12_AFR,3.0 +R12_RCPA,3.0 +R12_EEU,3.0 +R12_FSU,3.0 +R12_LAM,3.0 +R12_MEA,3.0 +R12_NAM,2.4 +R12_PAO,2.8 +R12_PAS,3.0 +R12_SAS,3.0 +R12_WEU,2.8 +R12_CHN,3.0 diff --git a/message_ix_models/model/config.py b/message_ix_models/model/config.py index a5318ecb31..d137c58118 100644 --- a/message_ix_models/model/config.py +++ b/message_ix_models/model/config.py @@ -6,7 +6,22 @@ @dataclass class Config: - """Settings and valid values for :mod:`message_ix_models.model` and submodules.""" + """Settings and valid values for :mod:`message_ix_models.model` and submodules. + + For backwards compatibility, it is possible to access these on a :class:`Context` + using: + + .. code-block:: python + + c = Context() + c.regions = "R14" + + …however, it is best to access them explicitly as: + + .. code-block:: python + + c.model.regions = "R14" + """ #: The 'node' codelist (regional aggregation) to use. Must be one of the lists of #: nodes described at :doc:`/pkg-data/node`. diff --git a/message_ix_models/model/macro.py b/message_ix_models/model/macro.py new file mode 100644 index 0000000000..b8f05d7b0a --- /dev/null +++ b/message_ix_models/model/macro.py @@ -0,0 +1,163 @@ +"""Tools for calibrating MACRO for MESSAGEix-GLOBIOM. + +See :doc:`message-ix:macro` for *general* documentation on MACRO and MESSAGE-MACRO. This +module contains tools specifically for using these models with MESSAGEix-GLOBIOM. +""" +import logging +from functools import lru_cache +from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING, List, Literal, Mapping, Optional, Union + +import pandas as pd + +from message_ix_models.model.bare import get_spec +from message_ix_models.util import nodes_ex_world + +if TYPE_CHECKING: # pragma: no cover + from sdmx.model.v21 import Code + + from message_ix_models import Context + +log = logging.getLogger(__name__) + +#: Default set of commodities to include in :func:`generate`. +COMMODITY = ["i_therm", "i_spec", "rc_spec", "rc_therm", "transport"] + + +def generate( + parameter: Literal["aeei", "config", "depr", "drate", "lotol"], + context: "Context", + commodities: Union[List[str], List["Code"]] = COMMODITY, + value: Optional[float] = None, +) -> pd.DataFrame: + """Generate uniform data for one :mod:`message_ix.macro` `parameter`. + + :meth:`message_ix.Scenario.add_macro` expects as its `data` parameter a + :class:`dict` that maps certain MACRO parameter names (or the special name "config") + to :class:`.pandas.DataFrame`. This function generates data for those data frames. + + For the particular dimensions, generate automatically includes: + + - "node": All nodes in the node code list given by :func:`.nodes_ex_world`, for the + node list indicated by :attr:`.model.Config.regions`. + - "year": All periods from the period *before* the first model year. + - "commodity": The elements of `commodities`. + - "sector": If each entry of `commodities` is a :class:`.Code` and has an annotation + with id="macro-sector", the value of that annotation. Otherwise, the same as + `commodity`. + + `value` supplies the parameter value, which is the same for all observations. + The labels level="useful" and unit="-" are fixed. + + Parameters + ---------- + parameter : str + MACRO parameter for which to generate data. + context + Used with :func:`.bare.get_spec`. + commodities : list of str or Code + Commodities to include in the MESSAGE-MACRO linkage. + value : float + Parameter value. + + Returns + ------- + pandas.DataFrame + The columns vary according to `parameter`: + + - "aeei": node, sector, year, value, unit. + - "depr", "drate", or "lotol": node, value, unit. + - "config": node, sector, commodity, level, year. + """ + spec = get_spec(context) + + if isinstance(commodities[0], str): + c_codes = spec.add.set["commodity"] + else: + c_codes = commodities + + @lru_cache + def _sector(commodity: str) -> str: + try: + idx = c_codes.index(commodity) + return str(c_codes[idx].get_annotation(id="macro-sector").text) + except (KeyError, ValueError) as e: + log.info(e) + return str(commodity) + + # AEEI data must begin from the period before the first model period + y0_index = spec.add.set["year"].index(spec.add.y0) + iterables = dict( + c_s=zip( # Paired commodity and sector + map(str, commodities), map(_sector, commodities) + ), + level=["useful"], + node=nodes_ex_world(spec.add.N), + sector=map(_sector, commodities), + year=spec.add.set["year"][y0_index:], + ) + + if parameter == "aeei": + dims = ["node", "year", "sector"] + iterables.update(year=spec.add.set["year"][y0_index - 1 :]) + elif parameter == "config": + dims = ["node", "c_s", "level", "year"] + assert value is None + elif parameter in ("depr", "drate", "lotol"): + dims = ["node"] + else: + raise NotImplementedError(f"generate(…) for MACRO parameter {parameter!r}") + + result = pd.DataFrame( + [tuple(values) for values in product(*[iterables[d] for d in dims])], + columns=dims, + ) + + if parameter == "config": + return pd.concat( + [ + result.drop("c_s", axis=1), + pd.DataFrame(result["c_s"].tolist(), columns=["commodity", "sector"]), + ], + axis=1, + ) + else: + return result.assign(value=value, unit="-") + + +def load(base_path: Path) -> Mapping[str, pd.DataFrame]: + """Load MACRO data from CSV files. + + The function reads files in the simple/long CSV format understood by + :func:`genno.computations.load_file`. For use with + :meth:`~message_ix.Scenario.add_macro`, the dimension names should be given in full, + for instance "node" or "sector". + + Parameters + ---------- + base_path : pathlib.Path + Directory containing zero or more CSV files. + + Returns + ------- + dict of (str -> pandas.DataFrame) + Mapping from MACRO calibration parameter names to data; one entry for each file + in `base_path`. + """ + from genno.computations import load_file + + result = {} + for filename in base_path.glob("*.csv"): + name = filename.stem + + q = load_file(filename, name=name) + + result[name] = ( + q.to_frame() + .reset_index() + .rename(columns={name: "value"}) + .assign(unit=f"{q.units:~}" or "-") + ) + + return result diff --git a/message_ix_models/tests/model/test_macro.py b/message_ix_models/tests/model/test_macro.py new file mode 100644 index 0000000000..7affbe9391 --- /dev/null +++ b/message_ix_models/tests/model/test_macro.py @@ -0,0 +1,48 @@ +import pandas as pd +import pandas.testing as pdt +import pytest +from sdmx.model import Annotation, Code + +from message_ix_models.model.macro import generate, load +from message_ix_models.util import package_data_path + + +@pytest.mark.parametrize( + "parameter, value", + [ + ("aeei", 1.0), + ("config", None), + ("depr", 1.0), + ("drate", 1.0), + ("lotol", 1.0), + pytest.param("foo", 1.0, marks=pytest.mark.xfail(raises=NotImplementedError)), + ], +) +def test_generate0(test_context, parameter, value): + result = generate(parameter, test_context, value=value) + + assert not result.isna().any(axis=None) + + +def test_generate1(test_context): + commodities = [ + Code(id="foo", annotations=[Annotation(id="macro-sector", text="BAR")]), + Code(id="baz", annotations=[Annotation(id="macro-sector", text="QUX")]), + ] + + result = generate("config", test_context, commodities) + + assert {"foo", "baz"} == set(result["commodity"].unique()) + + # Only the identified sectors appear + assert {"BAR", "QUX"} == set(result["sector"].unique()) + + # Only 2 unique (commodity, sector) combinations appear + assert 2 == len(result[["commodity", "sector"]].drop_duplicates()) + + +def test_load(test_context): + result = load(package_data_path("test", "macro")) + assert {"kgdp"} == set(result.keys()) + pdt.assert_index_equal(pd.Index(["node", "value", "unit"]), result["kgdp"].columns) + assert not result["kgdp"].isna().any(axis=None) diff --git a/message_ix_models/workflow.py b/message_ix_models/workflow.py index 296f568667..d9d303088f 100644 --- a/message_ix_models/workflow.py +++ b/message_ix_models/workflow.py @@ -1,6 +1,6 @@ """Tools for modeling workflows.""" import logging -from typing import Callable, List, Optional, Union +from typing import Callable, List, Literal, Mapping, Optional, Tuple, Union from genno import Computer from ixmp.utils import parse_url @@ -61,7 +61,8 @@ def __init__( except (AttributeError, ValueError): if clone is not False: raise TypeError("target= must be supplied for clone=True") - self.platform_info = self.scenario_info = dict() + self.platform_info = dict() + self.scenario_info = dict() # Store the callback and options self.action = action @@ -154,7 +155,7 @@ def add_step( action: Optional[CallbackType] = None, replace=False, **kwargs, - ) -> WorkflowStep: + ) -> str: """Add a :class:`WorkflowStep` to the workflow. Parameters @@ -173,8 +174,8 @@ def add_step( Returns ------- - WorkflowStep - a reference to the added step, for optional further modification. + str + The same as `name`. Raises ------ @@ -189,10 +190,8 @@ def add_step( # Remove any existing step self.graph.pop(name, None) - # Add to the Computer - self.add_single(name, step, "context", base, strict=True) - - return step + # Add to the Computer; return the name of the added step + return self.add_single(name, step, "context", base, strict=True) def run(self, name_or_names: Union[str, List[str]]): """Run all workflow steps necessary to produce `name_or_names`. @@ -215,18 +214,12 @@ def truncate(self, name: str): KeyError if step `name` does not exist. """ - - def _recurse_info(kind: str, step_name: str): - """Traverse the graph looking for non-empty platform_info/scenario_info.""" - task = self.graph[step_name] - return getattr(task[0], f"{kind}_info") or _recurse_info(kind, task[2]) - - # Generate a new step that merely loads the scenario identified by `name` or - # its base + # Generate a new step that merely loads the scenario identified by `name` or its + # base step = WorkflowStep(None) - step.scenario_info = _recurse_info("scenario", name) + step.scenario_info.update(self.guess_target(name, "scenario")[0]) try: - step.platform_info = _recurse_info("platform", name) + step.platform_info.update(self.guess_target(name, "platform")[0]) except KeyError as e: if e.args[0] is None: raise RuntimeError( @@ -238,6 +231,29 @@ def _recurse_info(kind: str, step_name: str): # Replace the existing step self.add_single(name, step, "context", None) + def guess_target( + self, step_name: str, kind: Literal["platform", "scenario"] = "scenario" + ) -> Tuple[Mapping, str]: + """Traverse the graph looking for non-empty platform_info/scenario_info. + + Returns the info, and the step name containing it. Usually, this will identify + the name of the platform, model, and/or scenario that is received and acted upon + by `step_name`. This may not be the case if preceding workflow steps perform + clone steps that are not recorded in the `target` parameter to + :class:`WorkflowStep`. + + Parameters + ---------- + step_name : str + Initial step from which to work backwards. + kind : str, "platform" or "scenario" + Whether to look up :attr:`~WorkflowStep.platform_info` or + :attr:`~WorkflowStep.scenario_info`. + """ + task = self.graph[step_name] + i = getattr(task[0], f"{kind}_info") + return (i.copy(), step_name) if len(i) else self.guess_target(task[2], kind) + def solve(context, scenario, **kwargs): scenario.solve(**kwargs)