From 3cad57e61744af28dad5f382e9046d686751b03e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 15:47:45 +0200 Subject: [PATCH 01/13] Adjust and sort imports in .report --- message_ix_models/report/__init__.py | 1 + message_ix_models/report/cli.py | 4 ++-- message_ix_models/report/sim.py | 4 ++-- message_ix_models/report/util.py | 3 ++- message_ix_models/tests/report/test_computations.py | 2 +- message_ix_models/tests/test_report.py | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index 68e023a6e0..be1156ad06 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -13,6 +13,7 @@ from genno import Key from genno.compat.pyam import iamc as handle_iamc from message_ix import Reporter, Scenario + from message_ix_models import Context from message_ix_models.model.structure import get_codes from message_ix_models.util import local_data_path, private_data_path diff --git a/message_ix_models/report/cli.py b/message_ix_models/report/cli.py index 1c3bbcc2c0..b822b34b19 100644 --- a/message_ix_models/report/cli.py +++ b/message_ix_models/report/cli.py @@ -4,12 +4,12 @@ import click import yaml + +from message_ix_models.report import register, report from message_ix_models.util import local_data_path, private_data_path from message_ix_models.util._logging import mark_time from message_ix_models.util.click import common_params -from message_data.reporting import register, report - log = logging.getLogger(__name__) diff --git a/message_ix_models/report/sim.py b/message_ix_models/report/sim.py index dcfb8f1d29..d77ca88f4a 100644 --- a/message_ix_models/report/sim.py +++ b/message_ix_models/report/sim.py @@ -9,10 +9,10 @@ from ixmp.reporting import RENAME_DIMS from message_ix.models import MESSAGE_ITEMS from message_ix.reporting import Key, KeyExistsError, Quantity, Reporter -from message_ix_models import ScenarioInfo from pandas.api.types import is_scalar -from message_data.tools import silence_log +from message_ix_models import ScenarioInfo +from message_ix_models.util._logging import silence_log # Shorthand for MESSAGE_VARS, below diff --git a/message_ix_models/report/util.py b/message_ix_models/report/util.py index c4436df392..f4d13542f4 100644 --- a/message_ix_models/report/util.py +++ b/message_ix_models/report/util.py @@ -7,9 +7,10 @@ from genno.compat.pyam.util import collapse as genno_collapse from iam_units import registry from message_ix.reporting import Key, Reporter -from message_ix_models.util import eval_anno from sdmx.model.v21 import Code +from message_ix_models.util import eval_anno + log = logging.getLogger(__name__) diff --git a/message_ix_models/tests/report/test_computations.py b/message_ix_models/tests/report/test_computations.py index 1ff65b8abf..a8735076a9 100644 --- a/message_ix_models/tests/report/test_computations.py +++ b/message_ix_models/tests/report/test_computations.py @@ -1,7 +1,7 @@ import xarray as xr from genno import Quantity -from message_data.reporting.computations import compound_growth +from message_ix_models.report.computations import compound_growth def test_compound_growth(): diff --git a/message_ix_models/tests/test_report.py b/message_ix_models/tests/test_report.py index 1e8974e2fa..1cf073c13a 100644 --- a/message_ix_models/tests/test_report.py +++ b/message_ix_models/tests/test_report.py @@ -2,9 +2,9 @@ import pandas as pd import pandas.testing as pdt import pytest -from message_ix_models import testing -from message_data.reporting import prepare_reporter, report, util +from message_ix_models import testing +from message_ix_models.report import prepare_reporter, report, util # Minimal reporting configuration for testing MIN_CONFIG = { From d595f6d26c74ce41936efa0c741384f295a5e571 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 15:51:07 +0200 Subject: [PATCH 02/13] Migrate enhanced silence_log() from message_data --- message_ix_models/util/_logging.py | 39 +++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/message_ix_models/util/_logging.py b/message_ix_models/util/_logging.py index 65d5d228b0..580da22b06 100644 --- a/message_ix_models/util/_logging.py +++ b/message_ix_models/util/_logging.py @@ -13,21 +13,48 @@ "silence_log", ] +log = logging.getLogger(__name__) + @contextmanager -def silence_log(): - """Context manager to temporarily silence log output. +def silence_log(names=None, level=logging.ERROR): + """Context manager to temporarily quiet 1 or more loggers. + + Parameters + ---------- + names : str, optional + Space-separated names of loggers to quiet. + level : int, optional + Minimum level of log messages to allow. Examples -------- >>> with silence_log(): >>> log.warning("This message is not recorded.") """ - # Restore the log level at the end of the context - with preserve_log_level(): - # Set the level to a very high value - logging.getLogger(__name__.split(".")[0]).setLevel(100) + # Default: the top-level logger for the package containing this file + if names is None: + names = [__name__.split(".")[0], "message_data"] + elif isinstance(names, str): + names = [names] + + log.info(f"Set level={level} for logger(s): {' '.join(names)}") + + # Retrieve the logger objects + loggers = list(map(logging.getLogger, names)) + # Store their current levels + levels = [] + + try: + for logger in loggers: + levels.append(logger.getEffectiveLevel()) # Store the current levels + logger.setLevel(level) # Set the level yield + finally: + # Restore the levels + for logger, original_level in zip(loggers, levels): + logger.setLevel(original_level) + log.info("…restored.") @contextmanager From cc949fde583e015b560a5038541cafae6a83f443 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 16:30:39 +0200 Subject: [PATCH 03/13] Update and integrate .report documentation --- doc/api/report/default-config.rst | 2 +- doc/api/report/index.rst | 109 ++++++++++++++++++++++-------- doc/index.rst | 1 + 3 files changed, 83 insertions(+), 29 deletions(-) diff --git a/doc/api/report/default-config.rst b/doc/api/report/default-config.rst index 78dcb92439..e4bd7b09ac 100644 --- a/doc/api/report/default-config.rst +++ b/doc/api/report/default-config.rst @@ -1,5 +1,5 @@ Default reporting configuration ******************************* -.. literalinclude:: ../../../data/report/global.yaml +.. literalinclude:: ../../../message_ix_models/data/report/global.yaml :language: yaml diff --git a/doc/api/report/index.rst b/doc/api/report/index.rst index 7f18e1ed36..2305ea8c25 100644 --- a/doc/api/report/index.rst +++ b/doc/api/report/index.rst @@ -1,32 +1,35 @@ -Reporting -********* +Reporting (:mod:`~.message_ix_models.report`) +********************************************* .. contents:: :local: See also: -- ``global.yaml``, the :doc:`reporting/default-config`. +- ``global.yaml``, the :doc:`default-config`. - Documentation for :mod:`genno` (:doc:`genno:index`), :mod:`ixmp.reporting`, and :mod:`message_ix.reporting`. -- :doc:`/reference/tools/post_processing`, still in use. -- Documentation for reporting specific to certain model variants: - - - :doc:`/reference/model/transport/report` -- `“Reporting” project board `_ on GitHub for the initial development of these features. .. toctree:: :hidden: - reporting/default-config + default-config + +Not public: + +- `“Reporting” project board `_ on GitHub for the initial implementation of these features. +- :doc:`m-data:/reference/tools/post_processing`, still in use. +- Documentation for reporting specific to certain model variants: + + - :doc:`m-data:/reference/model/transport/report` Introduction ============ See :doc:`the discussion in the MESSAGEix docs ` about the stack. -In short, :mod:`message_ix` cannot contain reporting code that references ``coal_ppl``, because not every model built on the MESSAGE framework will have a technology with this name. -Any reporting specific to ``coal_ppl`` must be in :mod:`message_data`, since all models in the MESSAGEix-GLOBIOM family will have this technology. +In short, :mod:`message_ix` must not contain reporting code that references ``coal_ppl``, because not every model built on the MESSAGE framework will have a technology with this name. +Any reporting specific to ``coal_ppl`` must be in :mod:`message_ix_models`, since all models in the MESSAGEix-GLOBIOM family will have this technology. -The basic **design pattern** of :mod:`message_data.reporting` is: +The basic **design pattern** of :mod:`message_ix_models.report` is: - A ``global.yaml`` file (i.e. in `YAML `_ format) that contains a *concise* yet *explicit* description of the reporting computations needed for a MESSAGE-GLOBIOM model. - :func:`~.reporting.prepare_reporter` reads the file and a Scenario object, and uses it to populate a new Reporter. @@ -35,7 +38,7 @@ The basic **design pattern** of :mod:`message_data.reporting` is: Features ======== -By combining these genno, ixmp, message_ix, and message_data features, the following functionality is provided. +By combining these genno, ixmp, message_ix, and message_ix_models features, the following functionality is provided. .. note:: If any of this does not appear to work as advertised, file a bug! @@ -64,9 +67,10 @@ Units base: example_var:a-b-c units: kJ +Continuous reporting +==================== -Continous reporting -=================== +.. note:: This section is no longer current. The IIASA TeamCity build server is configured to automatically run the full (:file:`global.yaml`) reporting on the following scenarios: @@ -81,31 +85,40 @@ This takes place: The results are output to Excel files that are preserved and made available as 'build artifacts' via the TeamCity web interface. - API reference ============= -.. currentmodule:: message_data.reporting +.. currentmodule:: message_ix_models.report -.. automodule:: message_data.reporting +.. automodule:: message_ix_models.report :members: + .. autosummary:: + + prepare_reporter + register + report -Computations ------------- +Operators +--------- -.. currentmodule:: message_data.reporting.computations -.. automodule:: message_data.reporting.computations +.. currentmodule:: message_ix_models.report.computations +.. automodule:: message_ix_models.report.computations :members: - :mod:`message_data` provides the following: + :mod:`message_ix_models` provides the following: .. autosummary:: + from_url + get_ts gwp_factors + make_output_path + model_periods + remove_ts share_curtailment - Other computations are provided by: + Other operators are provided by: - :mod:`message_ix.reporting.computations` - :mod:`ixmp.reporting.computations` @@ -114,11 +127,51 @@ Computations Utilities --------- -.. currentmodule:: message_data.reporting.util -.. automodule:: message_data.reporting.util +.. currentmodule:: message_ix_models.report.util +.. automodule:: message_ix_models.report.util :members: + .. autosummary:: + + add_replacements + as_quantity + collapse + collapse_gwp_info + copy_ts -.. currentmodule:: message_data.reporting.cli -.. automodule:: message_data.reporting.cli + +Command-line interface +---------------------- + +.. currentmodule:: message_ix_models.report.cli +.. automodule:: message_ix_models.report.cli :members: + + +.. code-block:: + + $ mix-models report --help + + Usage: mix-models report [OPTIONS] [KEY] + + Postprocess results. + + KEY defaults to the comprehensive report 'message::default', but may also be + the name of a specific model quantity, e.g. 'output'. + + --config can give either the absolute path to a reporting configuration + file, or the stem (i.e. name without .yaml extension) of a file in + data/report. + + With --from-file, read multiple Scenario identifiers from FILE, and report + each one. In this usage, --output-path may only be a directory. + + Options: + --dry-run Only show what would be done. + --config TEXT Path or stem for reporting config file. [default: + global] + -L, --legacy Invoke legacy reporting. + -m, --module MODULES Add extra reporting for MODULES. + -o, --output PATH Write output to file instead of console. + --from-file FILE Report multiple Scenarios listed in FILE. + --help Show this message and exit. diff --git a/doc/index.rst b/doc/index.rst index 5c61e68894..157fbd0d2e 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -34,6 +34,7 @@ Among other tasks, the tools allow modelers to: api/model-emissions api/model-snapshot api/disutility + api/report/index api/tools api/util api/testing From ea1084ef009fde2e386ecd839a67c93e960d3808 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 16:31:11 +0200 Subject: [PATCH 04/13] Add .report.cli to "mix-models" CLI commands --- message_ix_models/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/message_ix_models/cli.py b/message_ix_models/cli.py index fc5f0162a0..33e0ddaebb 100644 --- a/message_ix_models/cli.py +++ b/message_ix_models/cli.py @@ -114,6 +114,7 @@ def debug(ctx): "message_ix_models.model.snapshot", "message_ix_models.model.structure", "message_ix_models.model.water.cli", + "message_ix_models.report.cli", ] try: From 5b17367ddbb4ac15eb3eeba8b8bd4b726e33a4fa Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 16:31:31 +0200 Subject: [PATCH 05/13] Capitalize TODOs in .report.computations --- message_ix_models/report/computations.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/message_ix_models/report/computations.py b/message_ix_models/report/computations.py index 99d2abc512..fd185b63a8 100644 --- a/message_ix_models/report/computations.py +++ b/message_ix_models/report/computations.py @@ -1,4 +1,4 @@ -"""Atomic reporting computations for MESSAGEix-GLOBIOM.""" +"""Atomic reporting operations for MESSAGEix-GLOBIOM.""" import itertools import logging from typing import Any, List, Optional, Set, Union @@ -46,7 +46,7 @@ def get_ts( Corresponds to :meth:`ixmp.Scenario.timeseries`. - .. todo:: move upstream, e.g. to :mod:`ixmp` alongside :func:`.store_ts`. + .. todo:: Move upstream, e.g. to :mod:`ixmp` alongside :func:`.store_ts`. """ filters = filters or dict() @@ -95,7 +95,7 @@ def make_output_path(config, name): def model_periods(y: List[int], cat_year: pd.DataFrame) -> List[int]: """Return the elements of `y` beyond the firstmodelyear of `cat_year`. - .. todo:: move upstream, to :mod:`message_ix`. + .. todo:: Move upstream, to :mod:`message_ix`. """ return list( filter( @@ -114,10 +114,10 @@ def remove_ts( ) -> None: """Remove all time series data from `scenario`. - .. todo:: improve to provide the option to remove only those periods in the model + .. todo:: Improve to provide the option to remove only those periods in the model horizon. - .. todo:: move upstream, e.g. to :mod:`ixmp` alongside :func:`.store_ts`. + .. todo:: Move upstream, e.g. to :mod:`ixmp` alongside :func:`.store_ts`. """ data = scenario.timeseries() N = len(data) @@ -149,7 +149,7 @@ def remove_ts( # def from_url(url: str) -> message_ix.Scenario: # """Return a :class:`message_ix.Scenario` given its `url`. # -# .. todo:: move upstream to :mod:`message_ix.reporting`. +# .. todo:: Move upstream to :mod:`message_ix.reporting`. # .. todo:: Create a similar method in :mod:`ixmp.reporting` to load and return # :class:`ixmp.TimeSeries` (or :class:`ixmp.Scenario`) given its `url`. # """ From 280535f519ec7f1be9dcba481e5cabfb88f6b552 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 16:31:43 +0200 Subject: [PATCH 06/13] Add #116 to doc/whatsnew --- doc/whatsnew.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 54682b7ed6..f9bd32094f 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -4,6 +4,7 @@ What's new Next release ============ +- New module :mod:`message_ix_models.report` for reporting (:pull:`116`). - Add documentation on :ref:`migrate-filter-repo` using :program:`git filter-repo` and helper scripts (:pull:`89`). v2023.7.26 From 8818d2583d3f3a00e7748462d9aa9e73f492f174 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 16:51:27 +0200 Subject: [PATCH 07/13] Work around python/mypy#15843 in lint CI workflow --- .github/workflows/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f0be667003..058a9d5d72 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -23,6 +23,7 @@ jobs: type-hint-packages: >- genno iam-units + "mypy < 1.5" "pint < 0.21" pytest sdmx1 From a44d9c96a2d94c813a78d780451d2222a68e6975 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 30 Aug 2023 17:00:47 +0200 Subject: [PATCH 08/13] Update test of silence_log() --- message_ix_models/tests/util/test_logging.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/message_ix_models/tests/util/test_logging.py b/message_ix_models/tests/util/test_logging.py index c9d7e1cfc8..f3be6907de 100644 --- a/message_ix_models/tests/util/test_logging.py +++ b/message_ix_models/tests/util/test_logging.py @@ -33,7 +33,11 @@ def test_silence_log(caplog): with silence_log(): log.warning(msg) - assert [] == caplog.messages + assert [ + "Set level=40 for logger(s): message_ix_models message_data", + "…restored.", + ] == caplog.messages + caplog.clear() # After the "with" block, logging is restored log.warning(msg) From e5d6fca85a0a8e6ca8e890f885ec2d6de501e3f3 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 31 Aug 2023 14:19:47 +0200 Subject: [PATCH 09/13] Adjust compound_growth for pandas 2.1.0 --- message_ix_models/report/computations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message_ix_models/report/computations.py b/message_ix_models/report/computations.py index fd185b63a8..9913d1ef4d 100644 --- a/message_ix_models/report/computations.py +++ b/message_ix_models/report/computations.py @@ -28,7 +28,7 @@ def compound_growth(qty: Quantity, dim: str) -> Quantity: # Compute intervals along `dim` # The value at index d is the duration between d and the next index d+1 c = qty.coords[dim] - dur = (c - c.shift({dim: 1})).fillna(0).shift({dim: -1}) + dur = (c - c.shift({dim: 1})).fillna(0).shift({dim: -1}).fillna(0) # - Raise the values of `qty` to the power of the duration. # - Compute cumulative product along `dim` from the first index. # - Shift, so the value at index d is the growth relative to the prior index d-1 From e65fceb0623b3ff98ee8e7edf09cdfe31004ae80 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 31 Aug 2023 14:24:08 +0200 Subject: [PATCH 10/13] Satisfy mypy in two places with genno 1.18.0 --- message_ix_models/report/util.py | 3 ++- message_ix_models/workflow.py | 2 +- pyproject.toml | 12 +++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/message_ix_models/report/util.py b/message_ix_models/report/util.py index f4d13542f4..0b5c8b9275 100644 --- a/message_ix_models/report/util.py +++ b/message_ix_models/report/util.py @@ -5,6 +5,7 @@ from dask.core import quote from genno import Quantity from genno.compat.pyam.util import collapse as genno_collapse +from genno.core.key import single_key from iam_units import registry from message_ix.reporting import Key, Reporter from sdmx.model.v21 import Code @@ -191,7 +192,7 @@ def copy_ts(rep: Reporter, other: str, filters: Optional[dict]) -> Key: k1 = rep.add("from_url", f"scenario {_id}", quote(other)) k2 = rep.add("get_ts", f"ts data {_id}", k1, filters) - return rep.add("store_ts", f"copy ts {_id}", "scenario", k2) + return single_key(rep.add("store_ts", f"copy ts {_id}", "scenario", k2)) def add_replacements(dim: str, codes: Iterable[Code]) -> None: diff --git a/message_ix_models/workflow.py b/message_ix_models/workflow.py index d9d303088f..55b6e02e17 100644 --- a/message_ix_models/workflow.py +++ b/message_ix_models/workflow.py @@ -191,7 +191,7 @@ def add_step( self.graph.pop(name, None) # Add to the Computer; return the name of the added step - return self.add_single(name, step, "context", base, strict=True) + return str(self.add_single(name, step, "context", base, strict=True)) def run(self, name_or_names: Union[str, List[str]]): """Run all workflow steps necessary to produce `name_or_names`. diff --git a/pyproject.toml b/pyproject.toml index eefff7b84f..f0ba798fc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,10 @@ requires = ["build", "setuptools-scm"] dynamic = ["version"] name = "message-ix-models" description = "Tools for the MESSAGEix-GLOBIOM family of models" -authors = [ - {name = "IIASA Energy, Climate, and Environment (ECE) Program"}, -] +authors = [{ name = "IIASA Energy, Climate, and Environment (ECE) Program" }] maintainers = [ - {name = "Paul Natsuo Kishimoto", email = "mail@paul.kishimoto.name"}, - {name = "Fridolin Glatter", email = "glatter@iiasa.ac.at"}, + { name = "Paul Natsuo Kishimoto", email = "mail@paul.kishimoto.name" }, + { name = "Fridolin Glatter", email = "glatter@iiasa.ac.at" }, ] readme = "README.rst" classifiers = [ @@ -28,7 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: R", "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis" + "Topic :: Scientific/Engineering :: Information Analysis", ] requires-python = ">=3.8" dependencies = [ @@ -36,7 +34,7 @@ dependencies = [ "colorama", # When the minimum is greater than the minimum via message_ix; e.g. # message_ix >= 3.4.0 → ixmp >= 3.4.0 → genno >= 1.6.0", - "genno >= 1.8.0", + "genno >= 1.18.1", "iam_units", "message_ix >= 3.4.0", "pooch", From 280cfdfcdf98a0fde89e4691dee83843eb2ea5e2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 31 Aug 2023 15:34:40 +0200 Subject: [PATCH 11/13] Adjust .report tests migrated from message_data --- message_ix_models/testing.py | 3 +++ message_ix_models/tests/test_report.py | 1 + 2 files changed, 4 insertions(+) diff --git a/message_ix_models/testing.py b/message_ix_models/testing.py index 2874661393..353c90aae6 100644 --- a/message_ix_models/testing.py +++ b/message_ix_models/testing.py @@ -134,6 +134,9 @@ def test_context(request, session_context): """A copy of :func:`session_context` scoped to one test function.""" ctx = deepcopy(session_context) + # Ensure there is a report key + ctx.setdefault("report", dict()) + yield ctx ctx.delete() diff --git a/message_ix_models/tests/test_report.py b/message_ix_models/tests/test_report.py index 1cf073c13a..b3a97b5202 100644 --- a/message_ix_models/tests/test_report.py +++ b/message_ix_models/tests/test_report.py @@ -28,6 +28,7 @@ def test_report_bare_res(request, test_context): # reporter.get(key) +@pytest.mark.xfail(raises=ModuleNotFoundError, reason="Requires message_data") def test_report_legacy(caplog, request, tmp_path, test_context): """Legacy reporting can be invoked through :func:`.report()`.""" # Create a target scenario From 0a5d2995f6e2fe81b7d380720b30d49ce3fcbd6f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 31 Aug 2023 15:54:46 +0200 Subject: [PATCH 12/13] Adjust package names and relative paths in .report --- message_ix_models/report/__init__.py | 52 +++++++++++++++++----------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index be1156ad06..870069a882 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -1,7 +1,7 @@ import logging -import sys from copy import deepcopy from functools import partial +from importlib import import_module from operator import itemgetter from pathlib import Path from typing import Callable, List, Optional, Tuple, Union @@ -16,7 +16,7 @@ from message_ix_models import Context from message_ix_models.model.structure import get_codes -from message_ix_models.util import local_data_path, private_data_path +from message_ix_models.util import local_data_path, package_data_path from message_ix_models.util._logging import mark_time from .util import add_replacements @@ -67,12 +67,12 @@ def iamc(c: Reporter, info): """ # FIXME the upstream key "variable" for the configuration is confusing; choose a # better name - from message_data.reporting.util import collapse + from message_ix_models.report.util import collapse # Common base_key = Key.from_str_or_key(info["base"]) - # Use message_data custom collapse() method + # Use message_ix_models custom collapse() method info.setdefault("collapse", {}) # Add standard renames @@ -129,7 +129,7 @@ def register(name_or_callback: Union[Callable, str]) -> Optional[str]: from message_ix.reporting import Reporter from message_ix_models import Context - from message_data.reporting import register + from message_ix_models.report import register def cb(rep: Reporter, ctx: Context): # Modify `rep` by calling its methods ... @@ -140,22 +140,29 @@ def cb(rep: Reporter, ctx: Context): Parameters ---------- name_or_callback - If a string, this may be a submodule of :mod:`.message_data`, in which case the - function :func:`message_data.{name}.report.callback` is used. Or, it may be a - fully-resolved package/module name, in which case :func:`{name}.callback` is + If a string, this may be a submodule of :mod:`.message_ix_models`, or + :mod:`message_data`, in which case the function + ``{message_data,message_ix_models}.{name}.report.callback`` is used. Or, it may + be a fully-resolved package/module name, in which case ``{name}.callback`` is used. If a callable (function), it is used directly. """ if isinstance(name_or_callback, str): # Resolve a string - try: - # …as a submodule of message_data - name = f"message_data.{name_or_callback}.report" - __import__(name) - except ImportError: - # …as a fully-resolved package/module name - name = name_or_callback - __import__(name) - callback = sys.modules[name].callback + for name in [ + # As a submodule of message_ix_models + f"message_ix_models.{name_or_callback}.report", + # As a submodule of message_data + f"message_data.{name_or_callback}.report", + # As a fully-resolved package/module name + name_or_callback, + ]: + try: + mod = import_module(name) + except ModuleNotFoundError: + continue + else: + break + callback = mod.callback else: callback = name_or_callback name = callback.__name__ @@ -244,7 +251,7 @@ def report(context: Context, *args, **kwargs): # Default arguments for genno-based reporting context.report.setdefault("key", "default") - context.report.setdefault("config", private_data_path("report", "global.yaml")) + context.report.setdefault("config", package_data_path("report", "global.yaml")) rep, key = prepare_reporter(context) @@ -363,8 +370,11 @@ def prepare_reporter( has_solution = scenario.has_solution() # Append the message_data computations - rep.require_compat("message_data.reporting.computations") - rep.require_compat("message_data.tools.gdp_pop") + rep.require_compat("message_ix_models.report.computations") + try: + rep.require_compat("message_data.tools.gdp_pop") + except ModuleNotFoundError: + pass # Currently in message_data # Handle `report/config` setting passed from calling code context.setdefault("report", dict()) @@ -381,7 +391,7 @@ def prepare_reporter( p = config.get("path") if p and not p.exists() and not p.is_absolute(): # Try to resolve relative to the data/ directory - p = private_data_path("report", p) + p = package_data_path("report", p) assert p.exists(), p config.update(path=p) From 46250f0ac0504f6e985a791149fe8b841e528b45 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 31 Aug 2023 16:57:08 +0200 Subject: [PATCH 13/13] Force use of "iamc:" genno config handler during migration - Remove deprecated Key.from_str_or_key(). --- message_ix_models/report/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index 870069a882..c6cf4739db 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -70,7 +70,7 @@ def iamc(c: Reporter, info): from message_ix_models.report.util import collapse # Common - base_key = Key.from_str_or_key(info["base"]) + base_key = Key(info["base"]) # Use message_ix_models custom collapse() method info.setdefault("collapse", {}) @@ -376,6 +376,13 @@ def prepare_reporter( except ModuleNotFoundError: pass # Currently in message_data + # Force re-installation of the function iamc() in this file as the handler for + # "iamc:" sections in global.yaml. Until message_data.reporting is removed, then + # importing it will cause the iamc() function in *that* file to override the one + # registered above. + # TODO Remove, once message_data.reporting is removed. + genno.config.handles("iamc")(iamc) + # Handle `report/config` setting passed from calling code context.setdefault("report", dict()) context.report.setdefault("config", dict())