diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index a3e43d4a7e..8acafebb8f 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -110,6 +110,7 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + with: { python-version: "3.11" } - name: Force recreation of pre-commit virtual environment for mypy if: github.event_name == 'schedule' # Comment this line to run on a PR diff --git a/.gitignore b/.gitignore index 164685bf4e..e2513b3257 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,8 @@ dmypy.json # Generated and temporary data files debug/ cache/ +# LibreOffice lock files +.~lock*# # VSCode settings .vscode diff --git a/doc/Makefile b/doc/Makefile index d4bb2cbb9e..be3a3c83de 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -12,9 +12,12 @@ BUILDDIR = _build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help Makefile +clean-autosummary: + find . -name "$(BUILDDIR)" -prune -o -iname _autosummary -print0 | xargs -0 rmdir + +.PHONY: help clean-autosummary Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile +.DEFAULT: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/api/report/index.rst b/doc/api/report/index.rst index 2773575212..20718baf54 100644 --- a/doc/api/report/index.rst +++ b/doc/api/report/index.rst @@ -1,39 +1,54 @@ Reporting (:mod:`~.message_ix_models.report`) ********************************************* +On this page: + .. contents:: :local: -See also: +Elsewhere: - ``global.yaml``, the :doc:`default-config`. - Documentation for :mod:`genno` (:doc:`genno:index`), :mod:`ixmp.reporting`, and :mod:`message_ix.reporting`. +- Reporting of specific model variants: + + - :mod:`.water.reporting` + - (Private) :doc:`Reporting of message_data.model.transport ` .. toctree:: :hidden: default-config -Not public: - -- `“Reporting” project board `_ on GitHub for the initial implementation of these features. -- :doc:`m-data:/reference/tools/post_processing`, still in use. -- Documentation for reporting specific to certain model variants: - - - :doc:`m-data:/reference/model/transport/report` +.. _report-intro: Introduction ============ See :doc:`the discussion in the MESSAGEix docs ` about the stack. -In short, :mod:`message_ix` must not contain reporting code that references ``coal_ppl``, because not every model built on the MESSAGE framework will have a technology with this name. -Any reporting specific to ``coal_ppl`` must be in :mod:`message_ix_models`, since all models in the MESSAGEix-GLOBIOM family will have this technology. +In short, for instance: + +- :mod:`message_ix` **must not** contain reporting code that references :py:`technology="coal_ppl"`, because not every model built on the MESSAGE framework will have a technology with this name. +- Any model in the MESSAGEix-GLOBIOM family—built with :mod:`message_ix_models` and/or :mod:`message_data`—**should**, with few exceptions, have a :py:`technology="coal_ppl"`, since this appears in the common list of :ref:`technology-yaml`. + Reporting specific to this technology ID, *as it is represented* in this model family, should be in :mod:`message_ix_models` or user code. The basic **design pattern** of :mod:`message_ix_models.report` is: -- A ``global.yaml`` file (i.e. in `YAML `_ format) that contains a *concise* yet *explicit* description of the reporting computations needed for a MESSAGE-GLOBIOM model. -- :func:`~.report.prepare_reporter` reads the file and a Scenario object, and uses it to populate a new Reporter. - This function mostly relies on the :doc:`configuration handlers ` built in to Genno to handle the different sections of the file. +- :func:`~.report.prepare_reporter` populates a new :class:`~.message_ix.Reporter` for a given |Scenario| with many keys to report all quantities of interest in a MESSAGEix-GLOBIOM–family model. +- This function relies on *callbacks* defined in multiple submodules to add keys and tasks for general or tailored reporting calculations and actions. + Additional modules **should** define callback functions and register them with :func:`~report.register` when they are to be used. + For example: + + 1. The module :mod:`message_ix_models.report.plot` defines :func:`.plot.callback` that adds standard plots to the Reporter. + 2. The module :mod:`message_data.model.transport.report` defines :func:`~.message_data.model.transport.report.callback` that adds tasks specific to MESSAGEix-Transport. + 3. The module :mod:`message_data.projects.navigate.report` defines :func:`~.message_data.projects.navigate.report.callback` that add tasks specific to the ‘NAVIGATE’ research project. + + The callback (1) is always registered, because these plots are always applicable and can be expected to function correctly for all models in the family. In contrast, (2) and (3) **should** only be registered and run for the specific model variants for which they are developed/intended. + + Modules with tailored reporting configuration **may** also be indicated on the :ref:`command line ` by using the :program:`-m/--modules` option: :program:`mix-models report -m model.transport`. + +- A file :file:`global.yaml` file (in `YAML `_ format) contains a description of some of the reporting computations needed for a MESSAGE-GLOBIOM model. + :func:`~.report.prepare_reporter` uses the :doc:`configuration handlers ` built into :mod:`genno` (and some extensions specific to :mod:`message_ix_models`) to handle the different sections of the file. Features ======== @@ -67,24 +82,6 @@ Units base: example_var:a-b-c units: kJ -Continuous reporting -==================== - -.. note:: This section is no longer current. - -The IIASA TeamCity build server is configured to automatically run the full (:file:`global.yaml`) reporting on the following scenarios: - -.. literalinclude:: ../../ci/report.yaml - :caption: :file:`ci/report.yaml` - :language: yaml - -This takes place: - -- every morning at 07:00 IIASA time, and -- for every commit on every pull request branch, *if* the branch name includes ``report`` anywhere, e.g. ``feature/improve-reporting``. - -The results are output to Excel files that are preserved and made available as 'build artifacts' via the TeamCity web interface. - API reference ============= @@ -170,6 +167,40 @@ Utilities collapse_gwp_info copy_ts +.. _report-legacy: +.. currentmodule:: message_ix_models.report.compat + +Compatibility with :mod:`.message_data` +--------------------------------------- + +.. automodule:: message_ix_models.report.compat + :members: + + :mod:`.message_data` contains :doc:`m-data:reference/tools/post_processing`. + This code predates :mod:`genno` and the stack of tools built on it (:ref:`described above `); these were designed to avoid issues with performance and extensibility in the older code. [1]_ + :mod:`.report.compat` prepares a Reporter to perform the same calculations as :mod:`message_data.tools.post_processing`, except using :mod:`genno`. + + .. warning:: This code is **under development** and **incomplete**. + It is not yet a full or exact replacement for the legacy reporting code. + Use with caution. + + Main API: + + .. autosummary:: + TECH_FILTERS + callback + prepare_techs + get_techs + + Utility functions: + + .. autosummary:: + inp + eff + emi + out + +.. _report-cli: Command-line interface ====================== @@ -210,7 +241,20 @@ Command-line interface Testing ======= - .. currentmodule:: message_ix_models.report.sim .. automodule:: message_ix_models.report.sim :members: + +Continuous reporting +-------------------- + +As part of the :ref:`test-suite`, reporting is run on the same events (pushes and daily schedule) on publicly-available :doc:`model snapshots `. +One goal of these tests *inter alia* is to ensure that adjustments and improvements to the reporting code do not disturb manually-verified model outputs. + +As part of the (private) :mod:`message_data` test suite, multiple workflows run on regular schedules; some of these include a combination of :mod:`message_ix_models`-based and :ref:`‘legacy’ reporting `. +These workflows: + +- Operate on specific scenarios within IIASA databases. +- Create files in CSV, Excel, and/or PDF formats that are that are preserved and made available as 'build artifacts' via the GitHub Actions web interface and API. + +.. [1] See a (non-public) `“Reporting” project board `_ on GitHub for details of the initial implementation of these features. diff --git a/doc/api/testing.rst b/doc/api/testing.rst index 7d55232f2b..9f8656e6b5 100644 --- a/doc/api/testing.rst +++ b/doc/api/testing.rst @@ -1,8 +1,8 @@ +.. currentmodule:: message_ix_models.testing + Test utilities and fixtures (:mod:`.testing`) ********************************************* -.. currentmodule:: message_ix_models.testing - :doc:`Fixtures `: .. autosummary:: diff --git a/doc/repro.rst b/doc/repro.rst index 366c646dad..f86d3a5dca 100644 --- a/doc/repro.rst +++ b/doc/repro.rst @@ -20,6 +20,8 @@ This is a Scenario that has the same *structure* (ixmp 'sets') as actual instanc Code that operates on the global model can be tested on the bare RES; if it works on that scenario, this is one indication (necessary, but not always sufficient) that it should work on fully-populated scenarios. Such tests are faster and lighter than testing on fully-populated scenarios, and make it easier to isolate errors in the code that is being tested. +.. _test-suite: + Test suite (:mod:`message_ix_models.tests`) =========================================== @@ -58,13 +60,16 @@ In either case: - Running the test suite with ``--local-cache`` causes the local cache to be populated, and this will affect subsequent runs. - The continuous integration (below) services don't preserve caches, so code always runs. +.. _ci: + Continuous testing ================== -The test suite (:mod:`message_ix_models.tests`) is run using GitHub Actions for new commits on the ``main`` branch, or on any branch associated with a pull request. - -Because it is closed-source and requires access to internal IIASA resources, including databases, continuous integration for :mod:`message_data` is handled by an internal server running `TeamCity `_: https://ene-builds.iiasa.ac.at/project/message (requires authorization) +The test suite (:mod:`message_ix_models.tests`) is run using GitHub Actions for new commits on the ``main`` branch; new commits on any branch associated with a pull request; and on a daily schedule. +These ensure that the code is functional and produces expected outputs, even as upstream dependencies evolve. +Workflow runs and their outputs can be viewed `here `__. +Because it is closed-source and requires access to internal IIASA resources, including databases, continuous integration for :mod:`.message_data` is handled by GitHub Actions `self-hosted runners `__ running on IIASA systems. .. _export-test-data: diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 11c4987d29..7d9c85aedb 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -1,8 +1,10 @@ What's new ********** -.. Next release -.. ============ +Next release +============ + +- Add :mod:`message_ix_models.report.compat` :ref:`for emulating legacy reporting ` (:pull:`134`). v2023.10.16 =========== diff --git a/message_ix_models/data/technology.yaml b/message_ix_models/data/technology.yaml index 78ee17c739..c814a996e5 100644 --- a/message_ix_models/data/technology.yaml +++ b/message_ix_models/data/technology.yaml @@ -1224,12 +1224,12 @@ gas_rc: name: gas_rc description: Gas heating in residential/commercial sector type: final - sector: gas + sector: residential/commercial input: [gas, secondary] output: [gas, final] -gas_t/d: - name: gas_t/d +gas_t_d: + name: gas_t_d description: Transmission/Distribution of gas type: final vintaged: TRUE @@ -1237,8 +1237,8 @@ gas_t/d: input: [gas, secondary] output: [gas, final] -gas_t/d_ch4: - name: gas_t/d_ch4 +gas_t_d_ch4: + name: gas_t_d_ch4 description: Transmission/Distribution of gas with CH4 mitigation type: final vintaged: TRUE diff --git a/message_ix_models/model/config.py b/message_ix_models/model/config.py index d137c58118..25165e9d41 100644 --- a/message_ix_models/model/config.py +++ b/message_ix_models/model/config.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, fields +from dataclasses import dataclass, field, fields from message_ix_models.model.structure import codelists from message_ix_models.util.context import _ALIAS @@ -39,6 +39,11 @@ class Config: #: :func:`.bare.get_spec`. res_with_dummies: bool = False + #: Default or preferred units for model quantities and reporting. + units: dict = field( + default_factory=lambda: {"energy": "GWa", "CO2 emissions": "Mt / a"} + ) + def check(self): """Check the validity of :attr:`regions`, :attr:`relations`, :attr:`years`.""" for attr, name in [ diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index c1f9601517..22b5725f86 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -146,20 +146,24 @@ def cb(rep: Reporter, ctx: Context): """ if isinstance(name_or_callback, str): # Resolve a string - for name in [ + candidates = [ # As a fully-resolved package/module name name_or_callback, # As a submodule of message_ix_models f"message_ix_models.{name_or_callback}.report", # As a submodule of message_data f"message_data.{name_or_callback}.report", - ]: + ] + mod = None + for name in candidates: try: mod = import_module(name) except ModuleNotFoundError: continue else: break + if mod is None: + raise ModuleNotFoundError(" or ".join(candidates)) callback = mod.callback else: callback = name_or_callback @@ -338,11 +342,13 @@ def prepare_reporter( rep = Reporter.from_scenario(scenario) has_solution = scenario.has_solution() - # Append the message_data computations + # Append the message_data operators rep.require_compat("message_ix_models.report.computations") try: + # TODO Replace usage of operators from this module in favour of .exo_data; then + # remove this line. rep.require_compat("message_data.tools.gdp_pop") - except ModuleNotFoundError: + except ModuleNotFoundError: # pragma: no cover pass # Currently in message_data # Force re-installation of the function iamc() in this file as the handler for @@ -364,6 +370,7 @@ def prepare_reporter( **deepcopy(context.report.genno_config), fail="raise" if has_solution else logging.NOTSET, ) + rep.configure(model=deepcopy(context.model)) # Apply callbacks for other modules which define additional reporting computations for callback in CALLBACKS: diff --git a/message_ix_models/report/compat.py b/message_ix_models/report/compat.py new file mode 100644 index 0000000000..ab86819d9f --- /dev/null +++ b/message_ix_models/report/compat.py @@ -0,0 +1,420 @@ +"""Compatibility code that emulates :mod:`.message_data` reporting.""" +import logging +from functools import partial +from itertools import chain, count +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional + +from genno import Key, Quantity, quote +from genno.core.key import iter_keys, single_key + +if TYPE_CHECKING: + from genno import Computer + from ixmp.reporting import Reporter + from sdmx.model.common import Code + + from message_ix_models import Context + +__all__ = [ + "TECH_FILTERS", + "callback", + "eff", + "emi", + "get_techs", + "inp", + "out", + "prepare_techs", +] + +log = logging.getLogger(__name__) + +#: Filters for determining subsets of technologies. +#: +#: Each value is a Python expression :func:`eval`'d in an environment containing +#: variables derived from the annotations on :class:`Codes <.Code>` for each technology. +#: If the expression evaluates to :obj:`True`, then the code belongs to the set +#: identified by the key. +#: +#: See also +#: -------- +#: get_techs +#: prepare_techs +#: +TECH_FILTERS = { + "gas all": "c_in == 'gas' and l_in in 'secondary final' and '_ccs' not in id", + "gas extra": "False", + # Residential and commercial + "trp coal": "sector == 'transport' and c_in == 'coal'", + "trp gas": "sector == 'transport' and c_in == 'gas'", + "trp foil": "sector == 'transport' and c_in == 'fueloil'", + "trp loil": "sector == 'transport' and c_in == 'lightoil'", + "trp meth": "sector == 'transport' and c_in == 'methanol'", + # Transport + "rc gas": "sector == 'residential/commercial' and c_in == 'gas'", +} + + +# Counter for anon() +_ANON = map(lambda n: Key(f"_{n}"), count()) + + +def anon(name: Optional[str] = None, dims: Optional[Key] = None) -> Key: + """Create an ‘anonymous’ :class:`.Key`, optionally with `dims` from another Key.""" + result = next(_ANON) if name is None else Key(name) + + return result.append(*getattr(dims, "dims", [])) + + +def get_techs(c: "Computer", prefix: str, kinds: Optional[str] = None) -> List[str]: + """Return a list of technologies. + + The list is assembled from lists in `c` with keys like "t::{prefix} {kind}", + with one `kind` for each space-separated item in `kinds`. If no `kinds` are + supplied, "t::{prefix}" is used. + + See also + -------- + prepare_techs + """ + _kinds = kinds.split() if kinds else [""] + return list(chain(*[c.graph[f"t::{prefix} {k}".rstrip()][0].data for k in _kinds])) + + +def make_shorthand_function( + base_name: str, to_drop: str, default_unit_key: Optional[str] = None +): + """Create a shorthand function for adding tasks to a :class:`.Reporter`.""" + _to_drop = to_drop.split() + + def func( + c: "Computer", + technologies: List[str], + *, + name: Optional[str] = None, + filters: Optional[dict] = None, + unit_key: Optional[str] = default_unit_key, + ) -> Key: + f"""Select data from "{base_name}:*" and apply units. + + The returned key sums the result over the dimensions {_to_drop!r}. + + Parameters + ---------- + technologies : + List of technology IDs to include. + name : str, *optional* + If given, the name of the resulting key. Default: a name like "_123" + generated with :func:`anon`. + filters : dict, *optional* + Additional filters for selecting data from "{base_name}:*". Keys are short + dimension names (for instance, "c" for "commodity"); values are lists of + IDs. + unit_key : str, *optional* + Key for units to apply to the result. Must appear in :attr:`.Config.units`. + """ + base = single_key(c.full_key(base_name)) + key = anon(name, dims=base) + + indexers = dict(t=technologies) + indexers.update(filters or {}) + + if unit_key: + c.add(key + "sel", "select", base, indexers=indexers) + c.add( + key, + "assign_units", + key + "sel", + units=c.graph["config"]["model"].units[unit_key], + sums=True, + ) + else: + c.add(key, "select", base, indexers=indexers, sums=True) + + # Return the partial sum over some dimensions + return key.drop(*_to_drop) + + return func + + +inp = make_shorthand_function("in", "c h ho l no t", "energy") +emi = make_shorthand_function("rel", "nr r t yr") +out = make_shorthand_function("out", "c h hd l nd t", "energy") + + +def eff( + c: "Computer", + technologies: List[str], + filters_in: Optional[dict] = None, + filters_out: Optional[dict] = None, +) -> Key: + """Throughput efficiency (input / output) for `technologies`. + + Equivalent to :meth:`PostProcess.eff`. + + Parameters + ---------- + filters_in : dict, *optional* + Passed as the `filters` parameter to :func:`inp`. + filters_out : dict, *optional* + Passed as the `filters` parameter to :func:`out`. + """ + # TODO Check whether append / drop "t" is necessary + num = c.graph.unsorted_key(inp(c, technologies, filters=filters_in).append("t")) + denom = c.graph.unsorted_key(out(c, technologies, filters=filters_out).append("t")) + assert isinstance(num, Key) + assert isinstance(denom, Key) + + key = anon(dims=num) + + c.add(key, "div", num, denom, sums=True) + + return key.drop("t") + + +def pe_w_ccs_retro( + c: "Computer", + t: str, + t_scrub: str, + k_share: Optional[Key], + filters: Optional[dict] = None, +) -> Key: + """Calculate primary energy use of technologies with scrubbers. + + Equivalent to :func:`default_tables._pe_wCCS_retro` at L129. + """ + ACT: Key = single_key(c.full_key("ACT")) + + k0 = out(c, [t_scrub]) + k1 = c.add(anon(), "mul", k0, k_share) if k_share else k0 + + k2 = anon(dims=ACT).drop("t") + c.add(k2, "select", ACT, indexers=dict(t=t), drop=True, sums=True) + + # TODO determine the dimensions to drop for the numerator + k3, *_ = iter_keys(c.add(anon(dims=k2), "div", k2.drop("yv"), k2, sums=True)) + assert_dims(c, k3) + + filters_out = dict(c=["electr"], l=["secondary"]) + k4 = eff(c, [t], filters_in=filters, filters_out=filters_out) + k5 = single_key(c.add(anon(), "mul", k3, k4)) + k6 = single_key(c.add(anon(dims=k5), "div", k1, k5)) + assert_dims(c, k6) + + return k6 + + +def prepare_techs(c: "Computer", technologies: List["Code"]) -> None: + """Prepare sets of technologies in `c`. + + For each `key` → `expr` in :data:`TECH_FILTERS` and each technology :class:`Code` + `t` in `technologies`: + + - Apply the filter expression `expr` to information about `t`. + - If the expression evaluates to :obj:`True`, add it to a list in `c` at "t::{key}". + + These lists of technologies can be used directly or retrieve with :func:`get_techs`. + """ + result: Mapping[str, List[str]] = {k: list() for k in TECH_FILTERS} + + warned = set() # Filters that raise some kind of Exception + + # Iterate over technologies + for t in technologies: + # Assemble information about `t` from its annotations + info: Dict[str, Any] = dict(id=t.id) + # Sector + info["sector"] = str(t.get_annotation(id="sector").text) + try: + # Input commodity and level + info["c_in"], info["l_in"] = t.eval_annotation("input") + except (TypeError, ValueError): + info["c_in"] = info["l_in"] = None + + # Iterate over keys and respective filters + for key, expr in TECH_FILTERS.items(): + try: + # Apply the filter to the `info` about `t` + if eval(expr, None, info) is True: + # Filter evaluates to True → add `t` to the list of labels for `key` + result[key].append(t.id) + except Exception as e: + # Warn about this filter, only once + if expr not in warned: + log.warning(f"{e!r} when evaluating {expr!r}") + warned.add(expr) + + # Add keys like "t::trp gas" corresponding to TECH_FILTERS["trp gas"] + for k, v in result.items(): + c.add(f"t::{k}", quote(sorted(v))) + + +def assert_dims(c: "Computer", *keys: Key): + """Check the dimensions of `keys` for an "add", "sub", or "div" task. + + This is a sanity check needed because :py:`c.add("name", "div", …)` does not (yet) + automatically infer the dimensions of the resulting key. This is in contrast to + :py:`c.add("name", "mul", …)`, which *does* infer. + + Use this function after manual construction of a key for a "add", "div", or "sub" + task, in order to ensure the key matches the dimensionality of the quantity that + will result from the task. + + .. todo:: Remove once handled upstream in :mod:`genno`. + """ + for key in keys: + task = c.graph[key] + expected = Key.product("foo", *task[1:]) + + op = f" {task[0].__name__} " + assert set(key.dims) == set(expected.dims), ( + f"Task should produce {op.join(repr(k) for k in task[1:])} = " + f"{str(expected).split(':')[1]}; key indicates {str(key).split(':')[1]}" + ) + + +def callback(rep: "Reporter", context: "Context") -> None: + """Partially duplicate the behaviour of :func:`.default_tables.retr_CO2emi`. + + Currently, this prepares the following keys and the necessary preceding + calculations: + + - "transport emissions full::iamc": data for the IAMC variable + "Emissions|CO2|Energy|Demand|Transportation|Road Rail and Domestic Shipping" + """ + from message_ix_models.model.bare import get_spec + + from . import iamc + + N = len(rep.graph) + + # Structure information + spec = get_spec(context) + prepare_techs(rep, spec.add.set["technology"]) + + # Constants from report/default_units.yaml + rep.add("conv_c2co2:", 44.0 / 12.0) # dimensionless + # “Carbon content of natural gas” + rep.add("crbcnt_gas:", Quantity(0.482, units="Mt / GWa / a")) + + # Shorthand for get_techs(rep, …) + techs = partial(get_techs, rep) + + def full(name: str) -> Key: + """Return the full key for `name`.""" + return single_key(rep.full_key(name)) + + # L3059 from message_data/tools/post_processing/default_tables.py + # "gas_{cc,ppl}_share": shares of gas_cc and gas_ppl in the summed output of both + k0 = out(rep, ["gas_cc", "gas_ppl"]) + for t in "gas_cc", "gas_ppl": + k1 = out(rep, [t]) + k2 = rep.add(Key(f"{t}_share", k1.dims), "div", k0, k1) + assert_dims(rep, single_key(k2)) + + # L3026 + # "in:*:nonccs_gas_tecs": Input to non-CCS technologies using gas at l=(secondary, + # final), net of output from transmission and distribution technologies. + c_gas = dict(c=["gas"]) + k0 = inp(rep, techs("gas", "all extra"), filters=c_gas) + k1 = out(rep, ["gas_t_d", "gas_t_d_ch4"], filters=c_gas) + k2 = rep.add(Key("in", k1.dims, "nonccs_gas_tecs"), "sub", k0, k1) + assert_dims(rep, single_key(k2)) + + # L3091 + # "Biogas_tot_abs": absolute output from t=gas_bio [energy units] + # "Biogas_tot": above converted to its CO₂ content = CO₂ emissions from t=gas_bio + # [mass/time] + Biogas_tot_abs = out(rep, ["gas_bio"], name="Biogas_tot_abs") + rep.add("Biogas_tot", "mul", Biogas_tot_abs, "crbcnt_gas", "conv_c2co2") + + # L3052 + # "in:*:all_gas_tecs": Input to all technologies using gas at l=(secondary, final), + # including those with CCS. + k0 = inp( + rep, + ["gas_cc_ccs", "meth_ng", "meth_ng_ccs", "h2_smr", "h2_smr_ccs"], + filters=c_gas, + ) + k1 = rep.add( + Key("in", k0.dims, "all_gas_tecs"), "add", full("in::nonccs_gas_tecs"), k0 + ) + assert_dims(rep, k1) + + # L3165 + # "Hydrogen_tot:*": CO₂ emissions from t=h2_mix [mass/time] + k0 = emi( + rep, + ["h2_mix"], + name="_Hydrogen_tot", + filters=dict(r="CO2_cc"), + unit_key="CO2 emissions", + ) + # NB Must alias here, otherwise full("Hydrogen_tot") below gets a larger set of + # dimensions than intended + rep.add(Key("Hydrogen_tot", k0.dims), k0) + + # L3063 + # "in:*:nonccs_gas_tecs_wo_ccsretro": "in:*:nonccs_gas_tecs" minus inputs to + # technologies fitted with CCS add-on technologies. + filters = dict(c=["gas"], l=["secondary"]) + pe_w_ccs_retro_keys = [ + pe_w_ccs_retro(rep, *args, filters=filters) + for args in ( + ("gas_cc", "g_ppl_co2scr", full("gas_cc_share")), + ("gas_ppl", "g_ppl_co2scr", full("gas_ppl_share")), + # FIXME Raises KeyError + # ("gas_htfc", "gfc_co2scr", None), + ) + ] + k0 = rep.add(anon(dims=pe_w_ccs_retro_keys[0]), "add", *pe_w_ccs_retro_keys) + k1 = rep.add( + Key("in", k0.dims, "nonccs_gas_tecs_wo_ccsretro"), + "sub", + full("in::nonccs_gas_tecs"), + k0, + ) + assert_dims(rep, k0, k1) + + # L3144, L3234 + # "Biogas_trp", "Hydrogen_trp": transportation shares of emissions savings from + # biogas production/use, and from hydrogen production, respectively. + # X_trp = X_tot * (trp input of gas / `other` inputs) + k0 = inp(rep, techs("trp gas"), filters=c_gas) + for name, other in ( + ("Biogas", full("in::all_gas_tecs")), + ("Hydrogen", full("in::nonccs_gas_tecs_wo_ccsretro")), + ): + k1 = rep.add(anon(dims=other), "div", k0, other) + k2 = rep.add(f"{name}_trp", "mul", f"{name}_tot", k1) + assert_dims(rep, single_key(k1)) + + # L3346 + # "FE_Transport": CO₂ emissions from all transportation technologies directly using + # fossil fuels. + FE_Transport = emi( + rep, + techs("trp", "coal foil gas loil meth"), + name="FE_Transport", + filters=dict(r=["CO2_trp"]), + unit_key="CO2 emissions", + ) + + # L3886 + # "Transport": CO₂ emissions from transport. "FE_Transport" minus emissions saved by + # use of biogas in transport, plus emissions from production of hydrogen used in + # transport. + k0 = rep.add(anon(dims=FE_Transport), "sub", FE_Transport, full("Biogas_trp")) + k1, *_ = iter_keys( + rep.add(Key("Transport", k0.dims), "add", k0, full("Hydrogen_trp"), sums=True) + ) + assert_dims(rep, k0, k1) + + # TODO Identify where to sum on "h", "m", "yv" dimensions + + # Convert to IAMC structure + var = "Emissions|CO2|Energy|Demand|Transportation|Road Rail and Domestic Shipping" + info = dict(variable="transport emissions", base=k1.drop("h", "m", "yv"), var=[var]) + iamc(rep, info) + + # TODO use store_ts() to store on scenario + + log.info(f"Added {len(rep.graph) - N} keys") diff --git a/message_ix_models/report/computations.py b/message_ix_models/report/computations.py index e85070d874..a00c34e044 100644 --- a/message_ix_models/report/computations.py +++ b/message_ix_models/report/computations.py @@ -15,6 +15,8 @@ from message_ix_models import Context if TYPE_CHECKING: + from pathlib import Path + from genno import Computer, Key from sdmx.model.v21 import Code @@ -68,7 +70,7 @@ def compound_growth(qty: Quantity, dim: str) -> Quantity: return pow(qty, Quantity(dur)).cumprod(dim).shift({dim: 1}).fillna(1.0) -@Operator.define +@Operator.define() def exogenous_data(): """No action. @@ -120,7 +122,7 @@ def get_ts( return scenario.timeseries(iamc=iamc, subannual=subannual, **filters) -def gwp_factors(): +def gwp_factors() -> Quantity: """Use :mod:`iam_units` to generate a Quantity of GWP factors. The quantity is dimensionless, e.g. for converting [mass] to [mass], andhas @@ -154,7 +156,7 @@ def gwp_factors(): ) -def make_output_path(config, name): +def make_output_path(config: Mapping, name: str) -> "Path": """Return a path under the "output_dir" Path from the reporter configuration.""" return config["output_dir"].joinpath(name) @@ -164,29 +166,28 @@ def model_periods(y: List[int], cat_year: pd.DataFrame) -> List[int]: .. todo:: Move upstream, to :mod:`message_ix`. """ - return list( - filter( - lambda year: cat_year.query("type_year == 'firstmodelyear'")["year"].item() - <= year, - y, - ) - ) + y0 = cat_year.query("type_year == 'firstmodelyear'")["year"].item() + return list(filter(lambda year: y0 <= year, y)) def remove_ts( scenario: ixmp.Scenario, - config: dict, + config: Optional[dict] = None, after: Optional[int] = None, dump: bool = False, ) -> None: """Remove all time series data from `scenario`. - .. todo:: Improve to provide the option to remove only those periods in the model - horizon. + Note that data stored with :meth:`.add_timeseries` using :py:`meta=True` as a + keyword argument cannot be removed using :meth:`.TimeSeries.remove_timeseries`, and + thus also not with this operator. - .. todo:: Move upstream, e.g. to :mod:`ixmp` alongside :func:`.store_ts`. + .. todo:: Move upstream, to :mod:`ixmp` alongside :func:`.store_ts`. """ - data = scenario.timeseries() + if dump: + raise NotImplementedError + + data = scenario.timeseries().drop("value", axis=1) N = len(data) count = f"{N}" @@ -206,30 +207,22 @@ def remove_ts( else: scenario.commit(f"Remove time series data ({__name__}.remove_all_ts)") - if dump: - raise NotImplementedError - # Non-weak references to objects to keep them alive _FROM_URL_REF: Set[Any] = set() -# def from_url(url: str) -> message_ix.Scenario: -# """Return a :class:`message_ix.Scenario` given its `url`. -# -# .. todo:: Move upstream to :mod:`message_ix.reporting`. -# .. todo:: Create a similar method in :mod:`ixmp.reporting` to load and return -# :class:`ixmp.TimeSeries` (or :class:`ixmp.Scenario`) given its `url`. -# """ -# s, mp = message_ix.Scenario.from_url(url) -# assert s is not None -# _FROM_URL_REF.add(s) -# _FROM_URL_REF.add(mp) -# return s - - -def from_url(url: str) -> ixmp.TimeSeries: - """Return a :class:`ixmp.TimeSeries` given its `url`.""" - ts, mp = ixmp.TimeSeries.from_url(url) + +def from_url(url: str, cls=ixmp.TimeSeries) -> ixmp.TimeSeries: + """Return a :class:`ixmp.TimeSeries` or subclass instance, given its `url`. + + .. todo:: Move upstream, to :mod:`ixmp.reporting`. + + Parameters + ---------- + cls : type, *optional* + Subclass to instantiate and return; for instance, |Scenario|. + """ + ts, mp = cls.from_url(url) assert ts is not None _FROM_URL_REF.add(ts) _FROM_URL_REF.add(mp) diff --git a/message_ix_models/report/sim.py b/message_ix_models/report/sim.py index 1aa678796a..d19c120b2b 100644 --- a/message_ix_models/report/sim.py +++ b/message_ix_models/report/sim.py @@ -244,6 +244,10 @@ def data_from_file(path: Path, *, name: str, dims: Sequence[str]) -> Quantity: cols = list(dims) + ["value", "unit"] tmp = ( pd.read_csv(path, engine="pyarrow") + # Drop a leading index column that appears in some files + # TODO Adjust .snapshot.unpack() to avoid generating this column; update + # data; then remove this call + .drop(columns="", errors="ignore") .set_axis(cols, axis=1) .set_index(cols[:-2]) ) diff --git a/message_ix_models/tests/report/test_compat.py b/message_ix_models/tests/report/test_compat.py new file mode 100644 index 0000000000..670777ddac --- /dev/null +++ b/message_ix_models/tests/report/test_compat.py @@ -0,0 +1,106 @@ +import logging + +from genno import Computer +from ixmp.testing import assert_logs + +from message_ix_models import ScenarioInfo +from message_ix_models.model.structure import get_codes +from message_ix_models.report import prepare_reporter +from message_ix_models.report.compat import ( + TECH_FILTERS, + callback, + get_techs, + prepare_techs, +) + +from ..test_report import MARK, ss_reporter + + +@MARK[1] +def test_compat(tmp_path, test_context): + import numpy.testing as npt + + rep = ss_reporter() + prepare_reporter(test_context, reporter=rep) + + rep.add("scenario", ScenarioInfo(model="Model name", scenario="Scenario name")) + + # Tasks can be added to the reporter + callback(rep, test_context) + + # Select a key + key = ( + "transport emissions full::iamc" # IAMC structure + # "Transport" # Top level + # "Hydrogen_trp" # Second level + # "inp_nonccs_gas_tecs_wo_CCSRETRO" # Third level + # "_26" # Fourth level + ) + + # commented: Show what would be done + # print(rep.describe(key)) + # rep.visualize(tmp_path.joinpath("visualize.svg"), key) + + # Calculation runs + result = rep.get(key) + + # print(result.to_string()) # For intermediate results + + df = result.as_pandas() # For pyam.IamDataFrame, which doesn't have .to_string() + + # commented: Display or save output + # print(df.to_string()) + # df.to_csv(tmp_path.joinpath("transport-emissions-full.csv"), index=False) + + # Check a specific value + # TODO Expand set of expected values + npt.assert_allclose( + df.query("region == 'R11_AFR' and year == 2020")["value"].item(), 54.0532 + ) + + +def test_prepare_techs(test_context): + from message_ix_models.model.bare import get_spec + + # Retrieve a spec with the default set of technologies + spec = get_spec(test_context) + technologies = spec.add.set["technology"] + + c = Computer() + prepare_techs(c, technologies) + + # Expected sets of technologies based on the default technology.yaml + assert { + "gas all": [ + "gas_cc", + "gas_ct", + "gas_fs", + "gas_hpl", + "gas_htfc", + "gas_i", + "gas_ppl", + "gas_rc", + "gas_t_d", + "gas_t_d_ch4", + "gas_trp", + "hp_gas_i", + "hp_gas_rc", + ], + "gas extra": [], + # Residential and commercial + "rc gas": ["gas_rc", "hp_gas_rc"], + # Transport + "trp coal": ["coal_trp"], + "trp foil": ["foil_trp"], + "trp gas": ["gas_trp"], + "trp loil": ["loil_trp"], + "trp meth": ["meth_fc_trp", "meth_ic_trp"], + } == {k: get_techs(c, k) for k in TECH_FILTERS} + + +def test_prepare_techs_filter_error(caplog, monkeypatch): + """:func:`.prepare_techs` logs warnings for invalid filters.""" + monkeypatch.setitem(TECH_FILTERS, "foo", "not a filter") + + with assert_logs(caplog, "SyntaxError('invalid syntax", at_level=logging.WARNING): + prepare_techs(Computer(), get_codes("technology")) diff --git a/message_ix_models/tests/report/test_computations.py b/message_ix_models/tests/report/test_computations.py index 84c223fa2c..ed775b2d16 100644 --- a/message_ix_models/tests/report/test_computations.py +++ b/message_ix_models/tests/report/test_computations.py @@ -1,10 +1,41 @@ import re +import ixmp +import message_ix import pandas as pd +import pandas.testing as pdt +import pytest import xarray as xr -from genno import Quantity +from genno import Computer, Quantity +from ixmp.testing import assert_logs +from message_ix.testing import make_dantzig -from message_ix_models.report.computations import compound_growth, filter_ts +from message_ix_models import ScenarioInfo +from message_ix_models.model.structure import get_codes +from message_ix_models.report.computations import ( + compound_growth, + filter_ts, + from_url, + get_ts, + gwp_factors, + make_output_path, + model_periods, + remove_ts, + share_curtailment, +) + +from ..test_report import MARK + + +@pytest.fixture +def c() -> Computer: + return Computer() + + +@pytest.fixture +def scenario(test_context): + mp = test_context.get_platform() + yield make_dantzig(mp) def test_compound_growth(): @@ -45,3 +76,91 @@ def test_filter_ts(): # Only the first match group in `expr` is preserved assert {"ar"} == set(result.variable.unique()) + + +@MARK[0] +def test_from_url(scenario): + full_url = f"ixmp://{scenario.platform.name}/{scenario.url}" + + # Operator runs + result = from_url(full_url) + # Result is of the default class + assert result.__class__ is ixmp.TimeSeries + # Same object was retrieved + assert scenario.url == result.url + + # Same, but specifying message_ix.Scenario + result = from_url(full_url, message_ix.Scenario) + assert result.__class__ is message_ix.Scenario + assert scenario.url == result.url + + +@MARK[0] +def test_get_remove_ts(caplog, scenario): + # get_ts() runs + result0 = get_ts(scenario) + pdt.assert_frame_equal(scenario.timeseries(), result0) + + # Can be used through a Computer + + c = Computer() + c.require_compat("message_ix_models.report.computations") + c.add("scenario", scenario) + + key = c.add("test1", "get_ts", "scenario", filters=dict(variable="GDP")) + result1 = c.get(key) + assert 3 == len(result1) + + # remove_ts() can be used through Computer + key = c.add("test2", "remove_ts", "scenario", "config", after=1964) + + # Task runs, logs + # NB this log message is incorrect, because ixmp's JDBCBackend is unable to delete + # data stored with "meta=True". Only 1 row is removed + with assert_logs(caplog, "Remove 2 of 6 (1964 <= year) rows of time series data"): + c.get(key) + + # See comment above; only one row is removed + assert 6 - 1 == len(scenario.timeseries()) + + # remove_ts() can be used directly + remove_ts(scenario) + + # All non-'meta' data were removed + assert 3 == len(scenario.timeseries()) + + +def test_gwp_factors(): + result = gwp_factors() + + assert ("gwp metric", "e", "e equivalent") == result.dims + + +def test_make_output_path(tmp_path, c): + # Configure a Computer, ensuring the output_dir configuration attribute is set + c.configure(output_dir=tmp_path) + + # Add a computation that invokes make_output_path + c.add("test", make_output_path, "config", "foo.csv") + + # Returns the correct path + assert tmp_path.joinpath("foo.csv") == c.get("test") + + +def test_model_periods(): + # Prepare input data + si = ScenarioInfo() + si.year_from_codes(get_codes("year/B")) + cat_year = pd.DataFrame(si.set["cat_year"], columns=["type_year", "year"]) + + # Operator runs + result = model_periods(si.set["year"], cat_year) + + assert isinstance(result, list) + assert all(isinstance(y, int) for y in result) + assert 2020 == min(result) + + +@pytest.mark.xfail(reason="Incomplete") +def test_share_curtailment(): + share_curtailment() diff --git a/message_ix_models/tests/test_report.py b/message_ix_models/tests/test_report.py index 07fbfd471c..2c2cb3d159 100644 --- a/message_ix_models/tests/test_report.py +++ b/message_ix_models/tests/test_report.py @@ -5,9 +5,10 @@ import pandas as pd import pandas.testing as pdt import pytest +from ixmp.testing import assert_logs from message_ix_models import ScenarioInfo, testing -from message_ix_models.report import prepare_reporter, report, util +from message_ix_models.report import prepare_reporter, register, report, util from message_ix_models.report.sim import add_simulated_solution from message_ix_models.util import package_data_path @@ -19,6 +20,10 @@ } MARK = ( + pytest.mark.xfail( + condition=version("message_ix") < "3.5", + reason="Not supported with message_ix < 3.5", + ), pytest.mark.xfail( condition=version("message_ix") < "3.6", raises=NotImplementedError, @@ -27,7 +32,23 @@ ) -@MARK[0] +def test_register(caplog): + # Exception raised for unfindable module + with pytest.raises(ModuleNotFoundError): + register("foo.bar") + + # Adding a callback of the same name twice triggers a log message + def _cb(*args): + pass + + register(_cb) + with assert_logs( + caplog, "Already registered: ._cb" + ): + register(_cb) + + +@MARK[1] def test_report_bare_res(request, test_context): """Prepare and run the standard MESSAGE-GLOBIOM reporting on a bare RES.""" scenario = testing.bare_res(request, test_context, solved=True) @@ -107,7 +128,7 @@ def test_report_legacy(caplog, request, tmp_path, test_context): ) -@MARK[0] +@MARK[1] @pytest.mark.parametrize("regions", ["R11"]) def test_apply_units(request, test_context, regions): test_context.regions = regions @@ -225,7 +246,7 @@ def ss_reporter(): return rep -@MARK[0] +@MARK[1] def test_add_simulated_solution(test_context, test_data_path): # Simulated solution can be added to an empty Reporter rep = ss_reporter() @@ -253,7 +274,7 @@ def test_add_simulated_solution(test_context, test_data_path): assert np.isclose(79.76478, value.item()) -@MARK[0] +@MARK[1] def test_prepare_reporter(test_context): rep = ss_reporter() N = len(rep.graph) diff --git a/message_ix_models/util/scenarioinfo.py b/message_ix_models/util/scenarioinfo.py index 8025ae1c03..2471331484 100644 --- a/message_ix_models/util/scenarioinfo.py +++ b/message_ix_models/util/scenarioinfo.py @@ -132,21 +132,16 @@ def from_url(cls, url: str) -> "ScenarioInfo": def yv_ya(self): """:class:`pandas.DataFrame` with valid ``year_vtg``, ``year_act`` pairs.""" if self._yv_ya is None: - first = self.y0 - - # Product of all years - yv = ya = self.set["year"] - - # Predicate for filtering years - def _valid(elem): - yv, ya = elem - return first <= yv <= ya - # - Cartesian product of all yv and ya. - # - Filter only valid years. # - Convert to data frame. - self._yv_ya = pd.DataFrame( - filter(_valid, product(yv, ya)), columns=["year_vtg", "year_act"] + # - Filter only valid years. + self._yv_ya = ( + pd.DataFrame( + product(self.set["year"], self.set["year"]), + columns=["year_vtg", "year_act"], + ) + .query("@self.y0 <= year_vtg <= year_act") + .reset_index(drop=True) ) return self._yv_ya diff --git a/pyproject.toml b/pyproject.toml index 94699ea3f0..7013cadfcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "colorama", # When the minimum is greater than the minimum via message_ix; e.g. # message_ix >= 3.4.0 → ixmp >= 3.4.0 → genno >= 1.6.0", - "genno >= 1.18.1", + "genno >= 1.20.0", "iam_units >= 2023.9.11", "message_ix >= 3.4.0", "pooch",