From bfa97ea5304ab598ec2f9a768735b64e41e1e7ef Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 5 Nov 2023 17:40:50 +0000 Subject: [PATCH 01/37] =?UTF-8?q?Rename=20.reporting=20=E2=86=92=20.report?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename .report.computations → .report.operator. - Adjust some usage/references. --- ixmp/__init__.py | 2 +- ixmp/cli.py | 5 ++--- ixmp/{reporting => report}/__init__.py | 0 ixmp/{reporting/computations.py => report/operator.py} | 3 ++- ixmp/{reporting => report}/reporter.py | 6 ++++-- ixmp/{reporting => report}/util.py | 2 +- ixmp/testing/__init__.py | 2 +- ixmp/testing/data.py | 2 +- ixmp/tests/{reporting => report}/__init__.py | 0 .../test_computations.py => report/test_operator.py} | 2 +- ixmp/tests/{reporting => report}/test_reporter.py | 4 ++-- 11 files changed, 15 insertions(+), 13 deletions(-) rename ixmp/{reporting => report}/__init__.py (100%) rename ixmp/{reporting/computations.py => report/operator.py} (98%) rename ixmp/{reporting => report}/reporter.py (96%) rename ixmp/{reporting => report}/util.py (97%) rename ixmp/tests/{reporting => report}/__init__.py (100%) rename ixmp/tests/{reporting/test_computations.py => report/test_operator.py} (98%) rename ixmp/tests/{reporting => report}/test_reporter.py (98%) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index 1c745a663..2801240ba 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -10,7 +10,7 @@ from ixmp.model.base import ModelError from ixmp.model.dantzig import DantzigModel from ixmp.model.gams import GAMSModel -from ixmp.reporting import Reporter +from ixmp.report import Reporter from ixmp.utils import show_versions __all__ = [ diff --git a/ixmp/cli.py b/ixmp/cli.py index a7802227f..7bf704440 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -86,9 +86,8 @@ def main(ctx, url, platform, dbprops, model, scenario, version): @click.pass_obj def report(context, config, key): """Run reporting for KEY.""" - # Import here to avoid importing reporting dependencies when running - # other commands - from ixmp.reporting import Reporter + # Import here to avoid importing reporting dependencies when running other commands + from ixmp import Reporter if not context: raise click.UsageError( diff --git a/ixmp/reporting/__init__.py b/ixmp/report/__init__.py similarity index 100% rename from ixmp/reporting/__init__.py rename to ixmp/report/__init__.py diff --git a/ixmp/reporting/computations.py b/ixmp/report/operator.py similarity index 98% rename from ixmp/reporting/computations.py rename to ixmp/report/operator.py index d43dc7660..99f706bc7 100644 --- a/ixmp/reporting/computations.py +++ b/ixmp/report/operator.py @@ -6,9 +6,10 @@ from genno.core.quantity import Quantity from genno.util import parse_units -from ixmp.reporting.util import RENAME_DIMS, dims_for_qty, get_reversed_rename_dims from ixmp.utils import to_iamc_layout +from .util import RENAME_DIMS, dims_for_qty, get_reversed_rename_dims + log = logging.getLogger(__name__) diff --git a/ixmp/reporting/reporter.py b/ixmp/report/reporter.py similarity index 96% rename from ixmp/reporting/reporter.py rename to ixmp/report/reporter.py index 4b8406558..c32943544 100644 --- a/ixmp/reporting/reporter.py +++ b/ixmp/report/reporter.py @@ -6,7 +6,9 @@ from genno.core.computer import Computer, Key from ixmp.core.scenario import Scenario -from ixmp.reporting.util import RENAME_DIMS, keys_for_quantity + +from . import operator +from .util import RENAME_DIMS, keys_for_quantity class Reporter(Computer): @@ -17,7 +19,7 @@ def __init__(self, *args, **kwargs): # Append ixmp.reporting.computations to the modules in which the Computer will # look up computations names. - self.require_compat("ixmp.reporting.computations") + self.require_compat(operator) @classmethod def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": diff --git a/ixmp/reporting/util.py b/ixmp/report/util.py similarity index 97% rename from ixmp/reporting/util.py rename to ixmp/report/util.py index a3397d410..ee2690fe6 100644 --- a/ixmp/reporting/util.py +++ b/ixmp/report/util.py @@ -36,7 +36,7 @@ def dims_for_qty(data): def keys_for_quantity(ix_type, name, scenario): """Return keys for *name* in *scenario*.""" - from .computations import data_for_quantity + from .operator import data_for_quantity # Retrieve names of the indices of the ixmp item, without loading the data dims = dims_for_qty(scenario.idx_names(name)) diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index d7e58a2f1..1cd62fed8 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -228,7 +228,7 @@ def protect_rename_dims(): values to :data:`RENAME_DIMS`. Using this fixture ensures that the environment for other tests is not altered. """ - from ixmp.reporting import RENAME_DIMS + from ixmp.report import RENAME_DIMS saved = deepcopy(RENAME_DIMS) # Probably just copy() is sufficient yield diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 45519a373..dc08f48a8 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -10,7 +10,7 @@ from ixmp import Platform, Scenario, TimeSeries from ixmp.backend import IAMC_IDX -from ixmp.reporting import Quantity +from ixmp.report import Quantity #: Common (model name, scenario name) pairs for testing. SCEN = { diff --git a/ixmp/tests/reporting/__init__.py b/ixmp/tests/report/__init__.py similarity index 100% rename from ixmp/tests/reporting/__init__.py rename to ixmp/tests/report/__init__.py diff --git a/ixmp/tests/reporting/test_computations.py b/ixmp/tests/report/test_operator.py similarity index 98% rename from ixmp/tests/reporting/test_computations.py rename to ixmp/tests/report/test_operator.py index 20ecb4d8e..0a98df957 100644 --- a/ixmp/tests/reporting/test_computations.py +++ b/ixmp/tests/report/test_operator.py @@ -11,7 +11,7 @@ from ixmp import Scenario from ixmp.model.dantzig import DATA as dantzig_data -from ixmp.reporting.computations import map_as_qty, store_ts, update_scenario +from ixmp.report.operator import map_as_qty, store_ts, update_scenario from ixmp.testing import DATA as test_data from ixmp.testing import assert_logs, make_dantzig diff --git a/ixmp/tests/reporting/test_reporter.py b/ixmp/tests/report/test_reporter.py similarity index 98% rename from ixmp/tests/reporting/test_reporter.py rename to ixmp/tests/report/test_reporter.py index f570fe883..34a76e54a 100644 --- a/ixmp/tests/reporting/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -6,8 +6,8 @@ from genno import ComputationError, configure import ixmp -from ixmp.reporting.reporter import Reporter -from ixmp.reporting.util import RENAME_DIMS +from ixmp.report.reporter import Reporter +from ixmp.report.util import RENAME_DIMS from ixmp.testing import add_test_data, assert_logs, make_dantzig pytestmark = pytest.mark.usefixtures("parametrize_quantity_class") From 40c364cb7a7b8ad91a79ec28a1f80470276f7035 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 5 Nov 2023 17:49:28 +0000 Subject: [PATCH 02/37] Format utils/__init__.py --- ixmp/utils/__init__.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ixmp/utils/__init__.py b/ixmp/utils/__init__.py index 7361a723f..3bb303ff5 100644 --- a/ixmp/utils/__init__.py +++ b/ixmp/utils/__init__.py @@ -138,14 +138,17 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: ) # Merge the data from each side - yield current_name, pd.merge( - left, - right, - how="outer", - **on_arg, - sort=True, - suffixes=("_a", "_b"), - indicator=True, + yield ( + current_name, + pd.merge( + left, + right, + how="outer", + **on_arg, + sort=True, + suffixes=("_a", "_b"), + indicator=True, + ), ) # Maybe advance each iterators @@ -485,10 +488,11 @@ def describe(df): return pd.Series(result) + # group_keys silences a warning in pandas 1.5.0 info = ( platform.scenario_list(model=model, scen=scenario, default=default_only) - # group_keys silences a warning in pandas 1.5.0 - .groupby(["model", "scenario"], group_keys=True).apply(describe) + .groupby(["model", "scenario"], group_keys=True) + .apply(describe) ) if len(info): From d4e016fbb1572584b0a42a7b6457899c416e07f5 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Sun, 5 Nov 2023 17:50:25 +0000 Subject: [PATCH 03/37] Add .utils.DeprecatedPathFinder - Warn about imports from deprecated/old module names. - Preserve backwards-compatible imports downstream. --- ixmp/__init__.py | 14 +++++++++- ixmp/utils/__init__.py | 61 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index 2801240ba..a1a4ffdf8 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -1,4 +1,5 @@ import logging +import sys from importlib.metadata import PackageNotFoundError, version from ixmp._config import config @@ -11,7 +12,7 @@ from ixmp.model.dantzig import DantzigModel from ixmp.model.gams import GAMSModel from ixmp.report import Reporter -from ixmp.utils import show_versions +from ixmp.utils import DeprecatedPathFinder, show_versions __all__ = [ "IAMC_IDX", @@ -32,6 +33,17 @@ # Package is not installed __version__ = "999" +# Install a finder that locates modules given their old/deprecated names +sys.meta_path.append( + DeprecatedPathFinder( + __package__, + { + r"reporting(\..*)?": r"report\1", + "report.computations": "report.operator", + }, + ) +) + # Register Backends provided by ixmp BACKENDS["jdbc"] = JDBCBackend diff --git a/ixmp/utils/__init__.py b/ixmp/utils/__init__.py index 3bb303ff5..83a2062b9 100644 --- a/ixmp/utils/__init__.py +++ b/ixmp/utils/__init__.py @@ -2,8 +2,12 @@ import re import sys from contextlib import contextmanager +from functools import lru_cache +from importlib.abc import MetaPathFinder +from importlib.machinery import ModuleSpec, SourceFileLoader +from importlib.util import find_spec from pathlib import Path -from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, Mapping, Tuple from urllib.parse import urlparse from warnings import warn @@ -646,3 +650,58 @@ def update_par(scenario, name, data): if len(tmp): scenario.add_par(name, tmp) + + +class DeprecatedPathFinder(MetaPathFinder): + """Handle imports from deprecated module locations.""" + + map: Mapping[re.Pattern, str] + + def __init__(self, package: str, name_map: Mapping[str, str]): + # Prepend the package name to the source and destination + self.map = { + re.compile(rf"{package}\.{k}"): f"{package}.{v}" + for k, v in name_map.items() + } + + @lru_cache(maxsize=128) + def new_name(self, name): + # Apply each pattern in self.map successively + new_name = name + for pattern, repl in self.map.items(): + new_name = pattern.sub(repl, new_name) + + if name != new_name: + from warnings import warn + + warn( + f"Importing from {name!r} is deprecated and will fail in a future " + f"version. Use {new_name!r}.", + DeprecationWarning, + 2, + ) + + return new_name + + def find_spec(self, name, path, target=None): + new_name = self.new_name(name) + if new_name == name: + return None # No known transformation; let the importlib defaults handle. + + # Get an import spec for the module + spec = find_spec(new_name) + if not spec: + return None + + # Create a new spec that loads the module from its current location as if it + # were `name` + new_spec = ModuleSpec( + name=name, + # Create a new loader that loads from the actual file with the desired name + loader=SourceFileLoader(fullname=name, path=spec.origin), + origin=spec.origin, + ) + # These can't be passed through the constructor + new_spec.submodule_search_locations = spec.submodule_search_locations + + return new_spec From 094ed1f8337bcf4c337eb76058744b20e1ba763a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 6 Nov 2023 17:58:50 +0000 Subject: [PATCH 04/37] =?UTF-8?q?Rename=20ixmp.utils=20=E2=86=92=20ixmp.ut?= =?UTF-8?q?il?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Consistent with the rest of the message_ix stack. - Add to DeprecatedPathFinder. - Adjust references to satisfy mypy. --- ixmp/__init__.py | 12 +++++++++++- ixmp/backend/io.py | 2 +- ixmp/backend/jdbc.py | 7 +++++-- ixmp/cli.py | 2 +- ixmp/core/platform.py | 2 +- ixmp/core/scenario.py | 9 ++++++--- ixmp/core/timeseries.py | 6 +++--- ixmp/model/base.py | 2 +- ixmp/model/dantzig.py | 2 +- ixmp/model/gams.py | 2 +- ixmp/report/operator.py | 2 +- ixmp/{utils => util}/__init__.py | 0 ixmp/{utils => util}/sphinx_linkcode_github.py | 0 13 files changed, 32 insertions(+), 16 deletions(-) rename ixmp/{utils => util}/__init__.py (100%) rename ixmp/{utils => util}/sphinx_linkcode_github.py (100%) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index a1a4ffdf8..acfc57117 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -12,7 +12,7 @@ from ixmp.model.dantzig import DantzigModel from ixmp.model.gams import GAMSModel from ixmp.report import Reporter -from ixmp.utils import DeprecatedPathFinder, show_versions +from ixmp.util import DeprecatedPathFinder, show_versions __all__ = [ "IAMC_IDX", @@ -40,6 +40,7 @@ { r"reporting(\..*)?": r"report\1", "report.computations": "report.operator", + r"utils(\..*)?": r"util\1", }, ) ) @@ -64,3 +65,12 @@ handler.setLevel(logging.WARNING) log.addHandler(handler) log.setLevel(logging.WARNING) + + +def __getattr__(name): + if name == "utils": + import ixmp.util + + return ixmp.util + else: + raise AttributeError(name) diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index 20fb2c44c..bca14605d 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -3,7 +3,7 @@ import pandas as pd -from ixmp.utils import as_str_list, maybe_check_out, maybe_commit +from ixmp.util import as_str_list, maybe_check_out, maybe_commit from . import ItemType diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 4c6acd276..6001ce1f3 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -17,7 +17,7 @@ from ixmp.backend import FIELDS, ItemType from ixmp.backend.base import CachingBackend from ixmp.core.scenario import Scenario -from ixmp.utils import as_str_list, filtered +from ixmp.util import as_str_list, filtered log = logging.getLogger(__name__) @@ -660,7 +660,10 @@ def init(self, ts, annotation): # Call either newTimeSeries or newScenario method = getattr(self.jobj, "new" + klass) - jobj = method(ts.model, ts.scenario, *args) + try: + jobj = method(ts.model, ts.scenario, *args) + except java.IxException as e: + _raise_jexception(e) self._index_and_set_attrs(jobj, ts) diff --git a/ixmp/cli.py b/ixmp/cli.py index 7bf704440..33d759190 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -338,7 +338,7 @@ def list_platforms(): @click.pass_obj def list_scenarios(context, **kwargs): """List scenarios on the --platform.""" - from ixmp.utils import format_scenario_list + from ixmp.util import format_scenario_list if not context: raise click.UsageError( diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 2b6dc33db..9f7a8b86f 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -7,7 +7,7 @@ from ixmp._config import config from ixmp.backend import BACKENDS, FIELDS, ItemType -from ixmp.utils import as_str_list +from ixmp.util import as_str_list if TYPE_CHECKING: # pragma: no cover from ixmp.backend.base import Backend diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index fd08546e7..9e3009d49 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -12,7 +12,7 @@ from ixmp.core.platform import Platform from ixmp.core.timeseries import TimeSeries from ixmp.model import get_model -from ixmp.utils import as_str_list, check_year +from ixmp.util import as_str_list, check_year log = logging.getLogger(__name__) @@ -422,8 +422,11 @@ def items( continue # Retrieve the data, reducing the filters to only the dimensions of the item - yield name, self.par( - name, filters={k: v for k, v in filters.items() if k in idx_names} + yield ( + name, + self.par( + name, filters={k: v for k, v in filters.items() if k in idx_names} + ), ) def add_par( diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index be2c2462b..b0fe362d7 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -10,7 +10,7 @@ from ixmp.backend import FIELDS, IAMC_IDX, ItemType from ixmp.core.platform import Platform -from ixmp.utils import ( +from ixmp.util import ( as_str_list, maybe_check_out, maybe_commit, @@ -134,7 +134,7 @@ def from_url( Parameters ---------- url : str - See :meth:`parse_url `. + See :meth:`parse_url `. errors : 'warn' or 'raise' If 'warn', a failure to load the TimeSeries is logged as a warning, and the platform is still returned. If 'raise', the exception is raised. @@ -235,7 +235,7 @@ def transact( >>> # Changes to `ts` have been committed """ # TODO implement __enter__ and __exit__ to allow simpler "with ts: …" - from ixmp.utils import discard_on_error as discard_on_error_cm + from ixmp.util import discard_on_error as discard_on_error_cm if condition: maybe_check_out(self) diff --git a/ixmp/model/base.py b/ixmp/model/base.py index cce58b88b..c92da01f3 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -3,7 +3,7 @@ import re from abc import ABC, abstractmethod -from ixmp.utils import maybe_check_out, maybe_commit +from ixmp.util import maybe_check_out, maybe_commit log = logging.getLogger(__name__) diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index 2392c0018..d1255cc32 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -3,7 +3,7 @@ import pandas as pd -from ixmp.utils import maybe_check_out, maybe_commit, update_par +from ixmp.util import maybe_check_out, maybe_commit, update_par from .gams import GAMSModel diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 713f2c413..131cb25df 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -10,7 +10,7 @@ from ixmp.backend import ItemType from ixmp.model.base import Model, ModelError -from ixmp.utils import as_str_list +from ixmp.util import as_str_list log = logging.getLogger(__name__) diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index 99f706bc7..c714bd704 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -6,7 +6,7 @@ from genno.core.quantity import Quantity from genno.util import parse_units -from ixmp.utils import to_iamc_layout +from ixmp.util import to_iamc_layout from .util import RENAME_DIMS, dims_for_qty, get_reversed_rename_dims diff --git a/ixmp/utils/__init__.py b/ixmp/util/__init__.py similarity index 100% rename from ixmp/utils/__init__.py rename to ixmp/util/__init__.py diff --git a/ixmp/utils/sphinx_linkcode_github.py b/ixmp/util/sphinx_linkcode_github.py similarity index 100% rename from ixmp/utils/sphinx_linkcode_github.py rename to ixmp/util/sphinx_linkcode_github.py From 57b8e32e9d9f40f5bcc1a71bed90ddafa2253541 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 6 Nov 2023 18:02:30 +0000 Subject: [PATCH 05/37] =?UTF-8?q?Adjust=20further=20references=20.utils=20?= =?UTF-8?q?=E2=86=92=20.util?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASE_NOTES.rst | 8 ++++---- doc/api.rst | 8 ++++---- doc/conf.py | 4 ++-- ixmp/tests/core/test_timeseries.py | 2 +- ixmp/tests/report/test_reporter.py | 2 +- ixmp/tests/test_cli.py | 2 +- ixmp/tests/test_utils.py | 4 ++-- ixmp/util/__init__.py | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index aafecacbf..9068b4685 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -5,7 +5,7 @@ Next release .. ----------- - Support for Python 3.7 is dropped (:pull:`492`). -- New :func:`.utils.discard_on_error` and matching argument to :meth:`.TimeSeries.transact` to avoid locking :class:`.TimeSeries` / :class:`.Scenario` on failed operations with :class:`.JDBCBackend` (:pull:`488`). +- New :func:`.util.discard_on_error` and matching argument to :meth:`.TimeSeries.transact` to avoid locking :class:`.TimeSeries` / :class:`.Scenario` on failed operations with :class:`.JDBCBackend` (:pull:`488`). .. _v3.7.0: @@ -73,7 +73,7 @@ v3.4.0 (2022-01-24) Migration notes --------------- -:func:`ixmp.utils.isscalar` is deprecated. +:func:`ixmp.util.isscalar` is deprecated. Code should use :func:`numpy.isscalar`. All changes @@ -103,7 +103,7 @@ All changes - Add :mod:`genno` and :mod:`message_ix_models` to the output of :func:`show_versions` / ``ixmp show-versions`` (:pull:`416`). - Clean up test suite, improve performance, increase coverage (:pull:`416`). - Adjust documentation for deprecation of ``rixmp`` (:pull:`416`). -- Deprecate :func:`.utils.logger` (:pull:`399`). +- Deprecate :func:`.util.logger` (:pull:`399`). - Add a `quiet` option to :meth:`.GAMSModel.solve` and use in testing (:pull:`399`). - Fix :class:`.GAMSModel` would try to write GDX data to filenames containing invalid characters on Windows (:pull:`398`). - Format user-friendly exceptions when GAMSModel errors (:issue:`383`, :pull:`398`). @@ -141,7 +141,7 @@ ixmp v3.1.0 coincides with message_ix v3.1.0. - Fix a bug in :meth:`.read_excel` when parameter data is spread across multiple sheets (:pull:`345`). - Expand documentation and revise installation instructions (:pull:`363`). - Raise Python exceptions from :class:`.JDBCBackend` (:pull:`362`). -- Add :meth:`Scenario.items`, :func:`.utils.diff`, and allow using filters in CLI command ``ixmp export`` (:pull:`354`). +- Add :meth:`Scenario.items`, :func:`.util.diff`, and allow using filters in CLI command ``ixmp export`` (:pull:`354`). - Add functionality for storing ‘meta’ (annotations of model names, scenario names, versions, and some combinations thereof) (:pull:`353`). - Add :meth:`.Backend.add_model_name`, :meth:`~.Backend.add_scenario_name`, :meth:`~.Backend.get_model_names`, :meth:`~.Backend.get_scenario_names`, :meth:`~.Backend.get_meta`, :meth:`~.Backend.set_meta`, :meth:`~.Backend.remove_meta`. diff --git a/doc/api.rst b/doc/api.rst index f51f06835..fa5ccba34 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -197,9 +197,9 @@ To manipulate the configuration file, use the ``platform`` command in the ixmp c Utilities --------- -.. currentmodule:: ixmp.utils +.. currentmodule:: ixmp.util -.. automodule:: ixmp.utils +.. automodule:: ixmp.util :members: :exclude-members: as_str_list, check_year, isscalar, year_list, filtered @@ -218,7 +218,7 @@ Utilities Utilities for documentation ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: ixmp.utils.sphinx_linkcode_github +.. automodule:: ixmp.util.sphinx_linkcode_github :members: To use this extension, add it to the ``extensions`` setting in the Sphinx configuration file (usually :file:`conf.py`), and set the required ``linkcode_github_repo_slug``: @@ -227,7 +227,7 @@ Utilities for documentation extensions = [ ..., - "ixmp.utils.sphinx_linkcode_github", + "ixmp.util.sphinx_linkcode_github", ..., ] diff --git a/doc/conf.py b/doc/conf.py index 89d7d3984..cc12f80b6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -29,7 +29,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions coming # with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "ixmp.utils.sphinx_linkcode_github", + "ixmp.util.sphinx_linkcode_github", "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.coverage", @@ -110,7 +110,7 @@ "xarray": ("https://xarray.pydata.org/en/stable/", None), } -# -- Options for sphinx.ext.linkcode / ixmp.utils.sphinx_linkcode_github --------------- +# -- Options for sphinx.ext.linkcode / ixmp.util.sphinx_linkcode_github --------------- linkcode_github_repo_slug = "iiasa/ixmp" diff --git a/ixmp/tests/core/test_timeseries.py b/ixmp/tests/core/test_timeseries.py index 3f768b63c..50a6a6288 100644 --- a/ixmp/tests/core/test_timeseries.py +++ b/ixmp/tests/core/test_timeseries.py @@ -316,7 +316,7 @@ def test_remove(self, mp, ts, commit): assert ts.timeseries().empty def test_transact_discard(self, caplog, mp, ts): - caplog.set_level(logging.INFO, "ixmp.utils") + caplog.set_level(logging.INFO, "ixmp.util") df = expected(DATA[2050], ts) diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index 34a76e54a..7685c1395 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -101,7 +101,7 @@ def test_platform_units(test_mp, caplog, ureg): with caplog.at_level(logging.INFO): rep.get(x_key) - # NB cannot use assert_logs here. reporting.utils.parse_units uses the pint + # NB cannot use assert_logs here. report.util.parse_units uses the pint # application registry, so depending which tests are run and in which order, this # unit may already be defined. if len(caplog.messages): diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index 8d5fdeb57..dbd9d81e8 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -99,7 +99,7 @@ def test_list(ixmp_cli, test_mp): # 'list' without specifying a platform/scenario is a UsageError result = ixmp_cli.invoke(cmd) - assert result.exit_code == UsageError.exit_code + assert result.exit_code == UsageError.exit_code, (result.exception, result.output) # CLI works; nothing returned with a --match option that matches nothing result = ixmp_cli.invoke(["--platform", test_mp.name] + cmd) diff --git a/ixmp/tests/test_utils.py b/ixmp/tests/test_utils.py index 74c945b62..5e78edcc6 100644 --- a/ixmp/tests/test_utils.py +++ b/ixmp/tests/test_utils.py @@ -1,4 +1,4 @@ -"""Tests for ixmp.utils.""" +"""Tests for ixmp.util.""" import logging import numpy as np @@ -135,7 +135,7 @@ def test_diff_items(test_mp): def test_discard_on_error(caplog, test_mp): - caplog.set_level(logging.INFO, "ixmp.utils") + caplog.set_level(logging.INFO, "ixmp.util") # Create a test scenario, checked-in state s = make_dantzig(test_mp) diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index 83a2062b9..007a70246 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -41,7 +41,7 @@ def logger(): ixmp_logger.setLevel(logging.INFO) """ warn( - "ixmp.utils.logger() is deprecated as of 3.3.0, and will be removed in ixmp " + "ixmp.util.logger() is deprecated as of 3.3.0, and will be removed in ixmp " '5.0. Use logging.getLogger("ixmp").', DeprecationWarning, ) @@ -77,7 +77,7 @@ def as_str_list(arg, idx_names=None): def isscalar(x): """Returns True if `x` is a scalar.""" warn( - "ixmp.utils.isscalar() will be removed in ixmp >= 5.0. Use numpy.isscalar()", + "ixmp.util.isscalar() will be removed in ixmp >= 5.0. Use numpy.isscalar()", DeprecationWarning, ) return np.isscalar(x) From 7ea0dbb5fcd98e2657fe7ba15ddeb067b15a32e4 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 6 Nov 2023 18:03:08 +0000 Subject: [PATCH 06/37] Update test_store_ts() --- ixmp/tests/report/test_operator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ixmp/tests/report/test_operator.py b/ixmp/tests/report/test_operator.py index 0a98df957..08ef68330 100644 --- a/ixmp/tests/report/test_operator.py +++ b/ixmp/tests/report/test_operator.py @@ -148,7 +148,7 @@ def test_store_ts(request, caplog, test_mp): # A message is logged r = caplog.record_tuples[-1] assert ( - "ixmp.reporting.computations" == r[0] + "ixmp.report.operator" == r[0] and logging.ERROR == r[1] and r[2].startswith("Failed with ValueError('region = Moon')") ), caplog.record_tuples From 9686259b5966a98a343ae1e36154bb29b31136f1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 6 Nov 2023 18:03:52 +0000 Subject: [PATCH 07/37] Warn at the calling level in DeprecatedPathFinder --- ixmp/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index 007a70246..d10729e39 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -678,7 +678,7 @@ def new_name(self, name): f"Importing from {name!r} is deprecated and will fail in a future " f"version. Use {new_name!r}.", DeprecationWarning, - 2, + 3, ) return new_name From 791666de0a032c6cee8e556cebde31087a377397 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 6 Nov 2023 18:11:41 +0000 Subject: [PATCH 08/37] Update reporting docs --- doc/install.rst | 2 +- doc/reporting.rst | 74 ++++++++++++++++++++--------------------- ixmp/report/reporter.py | 4 +-- ixmp/report/util.py | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/doc/install.rst b/doc/install.rst index d7d0d7251..bc6b2bec4 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -71,7 +71,7 @@ GAMS (required) Graphviz (optional) ------------------- -:meth:`ixmp.reporting.Reporter.visualize` uses `Graphviz`_, a program for graph visualization. +:meth:`ixmp.Reporter.visualize` uses `Graphviz`_, a program for graph visualization. Installing ixmp causes the python :mod:`graphviz` package to be installed. If you want to use :meth:`.visualize` or run the test suite, the Graphviz program itself must also be installed; otherwise it is **optional**. diff --git a/doc/reporting.rst b/doc/reporting.rst index 57f5b63b7..2c03b8d46 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -1,9 +1,9 @@ Reporting / postprocessing ************************** -.. currentmodule:: ixmp.reporting +.. currentmodule:: ixmp.report -:mod:`ixmp.reporting` provides features for computing derived values from the contents of a :class:`ixmp.Scenario`, *after* it has been solved using a model and the solution data has been stored. +:mod:`ixmp.report` provides features for computing derived values from the contents of a :class:`ixmp.Scenario`, *after* it has been solved using a model and the solution data has been stored. It is built on the :mod:`genno` package, which has its own, separate documentation. This page provides only API documentation. @@ -17,10 +17,10 @@ This page provides only API documentation. Top-level classes and functions =============================== -.. automodule:: ixmp.reporting +.. automodule:: ixmp.report The following top-level objects from :mod:`genno` may also be imported from -:mod:`ixmp.reporting`. +:mod:`ixmp.report`. .. autosummary:: @@ -31,13 +31,13 @@ The following top-level objects from :mod:`genno` may also be imported from ~genno.core.quantity.Quantity ~genno.config.configure -:mod:`ixmp.reporting` additionally defines: +:mod:`ixmp.report` additionally defines: .. autosummary:: Reporter -.. autoclass:: ixmp.reporting.Reporter +.. autoclass:: ixmp.report.Reporter :members: :exclude-members: add_load_file @@ -79,9 +79,9 @@ The following top-level objects from :mod:`genno` may also be imported from Configuration ============= -:mod:`ixmp.reporting` adds handlers for two configuration sections, and modifies the behaviour of one from :mod:`genno` +:mod:`ixmp.report` adds handlers for two configuration sections, and modifies the behaviour of one from :mod:`genno` -.. automethod:: ixmp.reporting.filters +.. automethod:: ixmp.report.filters Reporter-specific configuration. @@ -104,7 +104,7 @@ Configuration # Clear existing filters for the "commodity" dimension commodity: null -.. automethod:: ixmp.reporting.rename_dims +.. automethod:: ixmp.report.rename_dims Reporter-specific configuration. @@ -116,19 +116,19 @@ Configuration rename_dims: i: i_renamed -.. automethod:: ixmp.reporting.units +.. automethod:: ixmp.report.units The only difference from :func:`genno.config.units` is that this handler keeps the configuration values stored in ``Reporter.graph["config"]``. This is so that :func:`.data_for_quantity` can make use of ``["units"]["apply"]`` -Computations -============ +Operators +========= -.. automodule:: ixmp.reporting.computations +.. automodule:: ixmp.report.operator :members: - :mod:`ixmp.reporting` defines these computations: + :mod:`ixmp.report` defines these operators: .. autosummary:: data_for_quantity @@ -136,34 +136,34 @@ Computations update_scenario store_ts - Basic computations are defined by :mod:`genno.computation`; and its compatibility modules; see there for details: + Basic operators are defined by :mod:`genno.operator` and its compatibility modules; see there for details: .. autosummary:: ~genno.compat.plotnine.Plot - ~genno.computations.add - ~genno.computations.aggregate - ~genno.computations.apply_units - ~genno.compat.pyam.computations.as_pyam - ~genno.computations.broadcast_map - ~genno.computations.combine - ~genno.computations.concat - ~genno.computations.disaggregate_shares - ~genno.computations.div - ~genno.computations.group_sum - ~genno.computations.interpolate - ~genno.computations.load_file - ~genno.computations.mul - ~genno.computations.pow - ~genno.computations.product - ~genno.computations.relabel - ~genno.computations.rename_dims - ~genno.computations.ratio - ~genno.computations.select - ~genno.computations.sum - ~genno.computations.write_report + ~genno.operator.add + ~genno.operator.aggregate + ~genno.operator.apply_units + ~genno.compat.pyam.operator.as_pyam + ~genno.operator.broadcast_map + ~genno.operator.combine + ~genno.operator.concat + ~genno.operator.disaggregate_shares + ~genno.operator.div + ~genno.operator.group_sum + ~genno.operator.interpolate + ~genno.operator.load_file + ~genno.operator.mul + ~genno.operator.pow + ~genno.operator.product + ~genno.operator.relabel + ~genno.operator.rename_dims + ~genno.operator.ratio + ~genno.operator.select + ~genno.operator.sum + ~genno.operator.write_report Utilities ========= -.. automodule:: ixmp.reporting.util +.. automodule:: ixmp.report.util :members: diff --git a/ixmp/report/reporter.py b/ixmp/report/reporter.py index c32943544..1019ddc48 100644 --- a/ixmp/report/reporter.py +++ b/ixmp/report/reporter.py @@ -17,8 +17,8 @@ class Reporter(Computer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # Append ixmp.reporting.computations to the modules in which the Computer will - # look up computations names. + # Append ixmp.report.operator to the modules in which the Computer will look up + # names self.require_compat(operator) @classmethod diff --git a/ixmp/report/util.py b/ixmp/report/util.py index ee2690fe6..caa6fa208 100644 --- a/ixmp/report/util.py +++ b/ixmp/report/util.py @@ -15,7 +15,7 @@ def dims_for_qty(data): If *data* is a :class:`pandas.DataFrame`, its columns are processed; otherwise it must be a list. - genno.RENAME_DIMS is used to rename dimensions. + :data:`RENAME_DIMS` is used to rename dimensions. """ if isinstance(data, pd.DataFrame): # List of the dimensions From d3d1b198fbad4db47ec3d0ac9bc2efd1e2ba338f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 6 Nov 2023 18:15:05 +0000 Subject: [PATCH 09/37] Add .. automodule:: ixmp to doc/api --- doc/api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index fa5ccba34..e91ee5cf9 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -3,6 +3,8 @@ Python (:mod:`ixmp` package) ============================ +.. automodule:: ixmp + The |ixmp| application programming interface (API) is organized around three classes: .. autosummary:: From 985398b9a5f19cb8a6ebfcc0dd56fc7a03d6c496 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 14 Nov 2023 14:14:07 +0100 Subject: [PATCH 10/37] Address Sphinx nitpicks in docstrings, docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - :class:`Foo` → :class:`.Foo`. - Don't use vague names ("array"). - Type with Literal[...] to make fixed values explicit. - Expand testing.__all__. - Patch outdated references in RELEASE_NOTES for usability. - Update Sphinx configuration: - napoleon_process_types to handle "optional", literals. - napoleon_type_aliases. - nitpick_ignore_regex. - rst_prolog - add :py:`…` role - intersphinx_mapping - add 5 packages. - Miscellaneous others. --- RELEASE_NOTES.rst | 32 +++++++++--------- doc/api-backend.rst | 24 +++++++++++--- doc/api.rst | 23 +++++++------ doc/cli.rst | 38 +++++++++++++++++++++ doc/conf.py | 62 +++++++++++++++++++++------------- doc/file-io.rst | 4 +-- doc/index.rst | 5 +-- doc/install.rst | 6 ++-- doc/reporting.rst | 61 +++++++++++++++++++--------------- ixmp/_config.py | 9 ++--- ixmp/backend/__init__.py | 6 ++-- ixmp/backend/base.py | 72 +++++++++++++++++++++++----------------- ixmp/backend/io.py | 8 ++--- ixmp/backend/jdbc.py | 13 +++++--- ixmp/cli.py | 1 + ixmp/core/platform.py | 14 ++++---- ixmp/core/scenario.py | 45 +++++++++++++------------ ixmp/core/timeseries.py | 30 +++++++++-------- ixmp/model/base.py | 26 +++++++++------ ixmp/model/dantzig.py | 2 +- ixmp/report/operator.py | 46 +++++++++++++++---------- ixmp/report/reporter.py | 10 +++--- ixmp/testing/__init__.py | 7 +++- ixmp/testing/data.py | 15 ++++++--- ixmp/testing/jupyter.py | 6 ++-- ixmp/testing/resource.py | 7 ++-- ixmp/util/__init__.py | 14 ++++---- 27 files changed, 358 insertions(+), 228 deletions(-) create mode 100644 doc/cli.rst diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 9068b4685..6d7a23667 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -34,8 +34,8 @@ All changes - Optionally tolerate failures to add individual items in :func:`.store_ts` reporting computation (:pull:`451`); use ``timeseries_only=True`` in check-out to function with :class:`.Scenario` with solution data stored. - Bugfix: :class:`.Config` squashed configuration values read from :file:`config.json`, if the respective keys were registered in downstream packages, e.g. :mod:`message_ix`. Allow the values loaded from file to persist (:pull:`451`). -- Adjust to genno 1.12 and set this as the minimum required version for :mod:`ixmp.reporting` (:pull:`451`). -- Add :meth:`.enforce` to the :class:`~.base.Model` API for enforcing structure/data consistency before :meth:`.Model.solve` (:pull:`450`). +- Adjust to genno 1.12 and set this as the minimum required version for :mod:`ixmp.reporting ` (:pull:`451`). +- Add :meth:`.enforce` to the :class:`~.base.Model` API for enforcing structure/data consistency before :meth:`.Model.run` (:pull:`450`). .. _v3.5.0: @@ -73,7 +73,7 @@ v3.4.0 (2022-01-24) Migration notes --------------- -:func:`ixmp.util.isscalar` is deprecated. +:py:`ixmp.util.isscalar()` is deprecated. Code should use :func:`numpy.isscalar`. All changes @@ -100,14 +100,14 @@ All changes ----------- - Add ``ixmp config show`` CLI command (:pull:`416`). -- Add :mod:`genno` and :mod:`message_ix_models` to the output of :func:`show_versions` / ``ixmp show-versions`` (:pull:`416`). +- Add :mod:`genno` and :mod:`message_ix_models` to the output of :func:`.show_versions` / ``ixmp show-versions`` (:pull:`416`). - Clean up test suite, improve performance, increase coverage (:pull:`416`). - Adjust documentation for deprecation of ``rixmp`` (:pull:`416`). - Deprecate :func:`.util.logger` (:pull:`399`). -- Add a `quiet` option to :meth:`.GAMSModel.solve` and use in testing (:pull:`399`). +- Add a `quiet` option to :class:`.GAMSModel` and use in testing (:pull:`399`). - Fix :class:`.GAMSModel` would try to write GDX data to filenames containing invalid characters on Windows (:pull:`398`). - Format user-friendly exceptions when GAMSModel errors (:issue:`383`, :pull:`398`). -- Adjust :mod:`ixmp.reporting` to use :mod:`genno` (:pull:`397`). +- Adjust :mod:`ixmp.reporting ` to use :mod:`genno` (:pull:`397`). - Fix two minor bugs in reporting (:pull:`396`). .. _v3.2.0: @@ -121,7 +121,7 @@ All changes - Increase JPype minimum version to 1.2.1 (:pull:`394`). - Adjust test suite for pandas v1.2.0 (:pull:`391`). - Raise clearer exceptions from :meth:`.add_par` for incorrect parameters; silently handle empty data (:pull:`374`). -- Depend on :mod:`openpyxl` instead of :mod:`xlrd` and :mod:`xlsxwriter` for Excel I/O; :mod:`xlrd` versions 2.0.0 and later do not support :file:`.xlsx` (:pull:`389`). +- Depend on :mod:`openpyxl` instead of :py:`xlrd` and :py:`xlsxwriter` for Excel I/O; :py:`xlrd` versions 2.0.0 and later do not support :file:`.xlsx` (:pull:`389`). - Add a parameter for exporting all model+scenario run versions to :meth:`.Platform.export_timeseries_data`, and fix a bug where exporting all runs happens uninteneded (:pull:`367`). - Silence noisy output from ignored exceptions on JDBCBackend/JVM shutdown (:pull:`378`). - Add a utility method, :func:`.gams_version`, to check the installed version of GAMS (:pull:`376`). @@ -141,18 +141,18 @@ ixmp v3.1.0 coincides with message_ix v3.1.0. - Fix a bug in :meth:`.read_excel` when parameter data is spread across multiple sheets (:pull:`345`). - Expand documentation and revise installation instructions (:pull:`363`). - Raise Python exceptions from :class:`.JDBCBackend` (:pull:`362`). -- Add :meth:`Scenario.items`, :func:`.util.diff`, and allow using filters in CLI command ``ixmp export`` (:pull:`354`). +- Add :meth:`.Scenario.items`, :func:`.util.diff`, and allow using filters in CLI command ``ixmp export`` (:pull:`354`). - Add functionality for storing ‘meta’ (annotations of model names, scenario names, versions, and some combinations thereof) (:pull:`353`). - Add :meth:`.Backend.add_model_name`, :meth:`~.Backend.add_scenario_name`, :meth:`~.Backend.get_model_names`, :meth:`~.Backend.get_scenario_names`, :meth:`~.Backend.get_meta`, :meth:`~.Backend.set_meta`, :meth:`~.Backend.remove_meta`. - Allow these to be called from :class:`.Platform` instances. - - Remove :meth:`.Scenario.delete_meta`. + - Remove :py:`Scenario.delete_meta()`. -- Avoid modifying indexers dictionary in :meth:`.AttrSeries.sel` (:pull:`349`). +- Avoid modifying indexers dictionary in :meth:`AttrSeries.sel ` (:pull:`349`). - Add region/unit parameters to :meth:`.Platform.export_timeseries_data` (:pull:`343`). - Preserve dtypes of index columns in :func:`.data_for_quantity` (:pull:`347`). - ``ixmp show-versions`` includes the path to the default JVM used by JDBCBackend/JPype (:pull:`339`). -- Make :class:`reporting.Quantity` classes interchangeable (:pull:`317`). +- Make :class:`reporting.Quantity ` classes interchangeable (:pull:`317`). - Use GitHub Actions for continuous testing and integration (:pull:`330`). .. _v3.0.0: @@ -203,18 +203,18 @@ All changes - Bump JPype dependency to 0.7.5 (:pull:`327`). - Improve memory management in :class:`.JDBCBackend` (:pull:`298`). -- Raise user-friendly exceptions from :meth:`.Reporter.get` in Jupyter notebooks and other read–evaluate–print loops (REPLs) (:pull:`316`). +- Raise user-friendly exceptions from :meth:`Reporter.get ` in Jupyter notebooks and other read–evaluate–print loops (REPLs) (:pull:`316`). - Ensure :meth:`.Model.initialize` is always called for new *and* cloned objects (:pull:`315`). - Add CLI command `ixmp show-versions` to print ixmp and dependency versions for debugging (:pull:`320`). - Bulk saving for metadata and exposing documentation AP (:pull:`314`)I -- Add :meth:`~.computations.apply_units`, :meth:`~computations.select` reporting calculations; expand :meth:`.Reporter.add` (:pull:`312`). -- :meth:`.Reporter.add_product` accepts a :class:`.Key` with a tag; :func:`~.computations.aggregate` preserves :class:`.Quantity` attributes (:pull:`310`). +- Add :func:`~.genno.operator.apply_units`, :func:`~.genno.operator.select` reporting operators; expand :meth:`Reporter.add ` (:pull:`312`). +- :func:`Reporter.add_product ` accepts a :class:`~.genno.Key` with a tag; :func:`~.genno.operator.aggregate` preserves :class:`~.genno.Quantity` attributes (:pull:`310`). - Add CLI command ``ixmp solve`` to run model solver (:pull:`304`). -- Add `dims` and `units` arguments to :meth:`Reporter.add_file`; remove :meth:`Reporter.read_config` (redundant with :meth:`Reporter.configure`) (:pull:`303`). +- Add `dims` and `units` arguments to :func:`Reporter.add_file `; remove :py:`Reporter.read_config()` (redundant with :meth:`Reporter.configure `) (:pull:`303`). - Add option to include `subannual` column in dataframe returned by :meth:`.TimeSeries.timeseries` (:pull:`295`). - Add :meth:`.Scenario.to_excel` and :meth:`.read_excel`; this functionality is transferred to ixmp from :mod:`message_ix` and enhanced for dealing with maximum row limits in Excel (:pull:`286`, :pull:`297`, :pull:`309`). - Include all tests in the ixmp package (:pull:`270`). -- Add :meth:`Model.initialize` API to help populate new Scenarios according to a model scheme (:pull:`212`). +- Add :meth:`.Model.initialize` API to help populate new Scenarios according to a model scheme (:pull:`212`). - Apply units to reported quantities (:pull:`267`). - Increase minimum pandas version to 1.0; adjust for `API changes and deprecations `_ (:pull:`261`). - Add :meth:`.export_timeseries_data` to write data for multiple scenarios to CSV (:pull:`243`). diff --git a/doc/api-backend.rst b/doc/api-backend.rst index 9957f507d..7872fe064 100644 --- a/doc/api-backend.rst +++ b/doc/api-backend.rst @@ -20,7 +20,7 @@ Provided backends .. currentmodule:: ixmp.backend.jdbc .. autoclass:: ixmp.backend.jdbc.JDBCBackend - :members: read_file, write_file + :members: handle_config, read_file, write_file JDBCBackend supports: @@ -59,7 +59,10 @@ Provided backends read_file write_file -.. automethod:: ixmp.backend.jdbc.start_jvm +.. autofunction:: ixmp.backend.jdbc.start_jvm + + +.. currentmodule:: ixmp.backend Backend API ----------- @@ -72,6 +75,7 @@ Backend API ixmp.backend.base.CachingBackend ixmp.backend.ItemType ixmp.backend.FIELDS + ixmp.backend.IAMC_IDX - :class:`ixmp.Platform` implements a *user-friendly* API for scientific programming. This means its methods can take many types of arguments, check, and transform them—in a way that provides modeler-users with easy, intuitive workflows. @@ -90,7 +94,7 @@ Backend API In the following, the bold-face words **required**, **optional**, etc. have specific meanings as described in `IETF RFC 2119 `_. Backend is an **abstract** class; this means it **must** be subclassed. - Most of its methods are decorated with :meth:`abc.abstractmethod`; this means they are **required** and **must** be overridden by subclasses. + Most of its methods are decorated with :any:`abc.abstractmethod`; this means they are **required** and **must** be overridden by subclasses. Others, marked below with "OPTIONAL:", are not so decorated. For these methods, the behaviour in the base Backend—often, nothing—is an acceptable default behaviour. @@ -191,13 +195,23 @@ Backend API :members: :private-members: - CachingBackend stores cache values for multiple :class:`.TimeSeries`/:class:`Scenario` objects, and for multiple values of a *filters* argument. + CachingBackend stores cache values for multiple :class:`.TimeSeries`/:class:`.Scenario` objects, and for multiple values of a *filters* argument. Subclasses **must** call :meth:`cache`, :meth:`cache_get`, and :meth:`cache_invalidate` as appropriate to manage the cache; CachingBackend does not enforce any such logic. + +.. automodule:: ixmp.backend + :members: FIELDS, IAMC_IDX + .. autoclass:: ixmp.backend.ItemType :members: :undoc-members: :member-order: bysource -.. autodata:: ixmp.backend.FIELDS +.. currentmodule:: ixmp.backend.io + +Common input/output routines for backends +----------------------------------------- + +.. automodule:: ixmp.backend.io + :members: diff --git a/doc/api.rst b/doc/api.rst index e91ee5cf9..a3d5debfd 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -74,6 +74,7 @@ TimeSeries commit discard_changes get_geodata + get_meta is_default last_update preload_timeseries @@ -82,6 +83,7 @@ TimeSeries remove_timeseries run_id set_as_default + set_meta timeseries transact url @@ -94,7 +96,7 @@ Scenario :show-inheritance: :members: - A Scenario is a :class:`TimeSeries` that also contains model data, including model solution data. + A Scenario is a :class:`.TimeSeries` that also contains model data, including model solution data. See the :ref:`data model documentation `. The Scenario class provides methods to manipulate model data items: @@ -115,7 +117,6 @@ Scenario clone equ equ_list - get_meta has_equ has_par has_set @@ -138,19 +139,11 @@ Scenario scalar set set_list - set_meta solve to_excel var var_list - -.. currentmodule:: ixmp.backend.io - -.. automodule:: ixmp.backend.io - :members: EXCEL_MAX_ROWS - - .. _configuration: Configuration @@ -195,6 +188,9 @@ To manipulate the configuration file, use the ``platform`` command in the ixmp c .. autoclass:: ixmp._config.Config :members: +.. autoclass:: ixmp._config.BaseValues + :members: + Utilities --------- @@ -206,6 +202,7 @@ Utilities :exclude-members: as_str_list, check_year, isscalar, year_list, filtered .. autosummary:: + diff discard_on_error format_scenario_list @@ -248,3 +245,9 @@ Utilities for testing .. automodule:: ixmp.testing :members: :exclude-members: pytest_report_header, pytest_sessionstart + :special-members: add_test_data + +.. currentmodule:: ixmp.testing.data + +.. automodule:: ixmp.testing.data + :members: DATA, HIST_DF, TS_DF diff --git a/doc/cli.rst b/doc/cli.rst new file mode 100644 index 000000000..911c7360d --- /dev/null +++ b/doc/cli.rst @@ -0,0 +1,38 @@ +Command-line interface +====================== + +:mod:`ixmp` has a **command-line** interface:: + + $ ixmp --help + Usage: ixmp [OPTIONS] COMMAND [ARGS]... + + Options: + --url ixmp://PLATFORM/MODEL/SCENARIO[#VERSION] + Scenario URL. + --platform TEXT Configured platform name. + --dbprops FILE Database properties file. + --model TEXT Model name. + --scenario TEXT Scenario name. + --version VERSION Scenario version. + --help Show this message and exit. + + Commands: + config Get and set configuration keys. + export Export scenario data to PATH. + import Import time series or scenario data. + list List scenarios on the --platform. + platform Configure platforms and storage backends. + report Run reporting for KEY. + show-versions Print versions of ixmp and its dependencies. + solve Solve a Scenario and store results on the Platform. + +The various commands allow to manipulate :ref:`configuration`, show debug and system information, invoke particular models and the :doc:`reporting` features. +The CLI is used as the basis for extended features provided by :mod:`message_ix`. + +.. currentmodule:: ixmp.cli + +CLI internals +------------- + +.. automodule:: ixmp.cli + :members: diff --git a/doc/conf.py b/doc/conf.py index cc12f80b6..89e2496ca 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -3,17 +3,6 @@ # This file only contains a selection of the most common options. For a full list see # the documentation: https://www.sphinx-doc.org/en/master/usage/configuration.html -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, add -# these directories to sys.path here. If the directory is relative to the documentation -# root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - # Import so that autodoc can find code import ixmp @@ -50,23 +39,27 @@ # html_extra_path. exclude_patterns = ["_build", "README.rst", "Thumbs.db", ".DS_Store"] +nitpick_ignore_regex = { + # These occur because there is no .. py:module:: directive for the *top-level* + # module or package in the respective documentation and inventories. + # TODO Remove once the respective docs are fixed + ("py:mod", "message_ix"), +} + # A string of reStructuredText that will be included at the beginning of every source # file that is read. version = ixmp.__version__ -rst_prolog = r""" -.. |MESSAGEix| replace:: MESSAGE\ :emphasis:`ix` - -.. |ixmp| replace:: :emphasis:`ix modeling platform` - -.. |version| replace:: {} +rst_prolog = rf""" +.. role:: py(code) + :language: python .. role:: strike - .. role:: underline -""".format( - version -) +.. |MESSAGEix| replace:: MESSAGE\ :emphasis:`ix` +.. |ixmp| replace:: :emphasis:`ix modeling platform` +.. |version| replace:: {version} +""" # -- Options for HTML output ----------------------------------------------------------- @@ -98,22 +91,45 @@ intersphinx_mapping = { "dask": ("https://docs.dask.org/en/stable/", None), - "genno": ("https://genno.readthedocs.io/en/latest", None), + "genno": ("https://genno.readthedocs.io/en/latest/", None), "jpype": ("https://jpype.readthedocs.io/en/latest", None), "message_ix": ("https://docs.messageix.org/en/latest/", None), + "message-ix-models": ( + "https://docs.messageix.org/projects/models/en/latest/", + None, + ), + "nbclient": ("https://nbclient.readthedocs.io/en/latest/", None), + "nbformat": ("https://nbformat.readthedocs.io/en/latest/", None), "numpy": ("https://numpy.org/doc/stable/", None), + "openpyxl": ("https://openpyxl.readthedocs.io/en/stable/", None), "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), "pint": ("https://pint.readthedocs.io/en/stable/", None), + "pyam": ("https://pyam-iamc.readthedocs.io/en/stable/", None), "python": ("https://docs.python.org/3/", None), "sparse": ("https://sparse.pydata.org/en/stable/", None), "sphinx": ("https://www.sphinx-doc.org/en/master/", None), "xarray": ("https://xarray.pydata.org/en/stable/", None), } -# -- Options for sphinx.ext.linkcode / ixmp.util.sphinx_linkcode_github --------------- +# -- Options for sphinx.ext.linkcode / ixmp.util.sphinx_linkcode_github ---------------- linkcode_github_repo_slug = "iiasa/ixmp" +# -- Options for sphinx.ext.napoleon --------------------------------------------------- + +napoleon_preprocess_types = True +napoleon_type_aliases = { + # Standard library + "callable": ":ref:`callable `", + "iterable": ":class:`collections.abc.Iterable`", + "sequence": ":class:`collections.abc.Sequence`", + # Upstream + "Quantity": ":class:`genno.Quantity`", + # This package + "Platform": "~ixmp.Platform", + "Scenario": "~ixmp.Scenario", +} + # -- Options for sphinx.ext.todo ------------------------------------------------------- # If true, `todo` and `todoList` produce output, else they produce nothing. diff --git a/doc/file-io.rst b/doc/file-io.rst index b40e437da..253b835aa 100644 --- a/doc/file-io.rst +++ b/doc/file-io.rst @@ -1,7 +1,7 @@ File formats and input/output ***************************** -In addition to the data management features provided by :doc:`api-backend`, ixmp is able to write and read :class:`TimeSeries` and :class:`Scenario` data to and from files. +In addition to the data management features provided by :doc:`api-backend`, ixmp is able to write and read :class:`.TimeSeries` and :class:`.Scenario` data to and from files. This page describes those options and formats. Time series data @@ -9,7 +9,7 @@ Time series data Time series data can be: -- Read using :meth:`.import_timeseries`, or the CLI command ``ixmp import timeseries FILE`` for a single TimeSeries object. +- Read using :meth:`.TimeSeries.read_file`, or the :doc:`CLI command ` ``ixmp import timeseries FILE`` for a single TimeSeries object. - Written using :meth:`.export_timeseries_data` for multiple TimeSeries objects at once. Both CSV and Excel files in the IAMC time-series format are supported. diff --git a/doc/index.rst b/doc/index.rst index 16488aeca..c41f1202e 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -33,10 +33,11 @@ Getting started install tutorials + cli - :doc:`install` - :doc:`tutorials` - +- :doc:`cli` Scientific programming API -------------------------- @@ -54,7 +55,7 @@ Scientific programming API reporting data-model -The `ixmp` has application programming interfaces (API) for efficient scientific workflows and data processing. +The `ixmp` has application programming interfaces (API) for scientific workflows and data processing. - :doc:`api` - :doc:`api-r` diff --git a/doc/install.rst b/doc/install.rst index bc6b2bec4..fa4de9680 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -71,9 +71,9 @@ GAMS (required) Graphviz (optional) ------------------- -:meth:`ixmp.Reporter.visualize` uses `Graphviz`_, a program for graph visualization. -Installing ixmp causes the python :mod:`graphviz` package to be installed. -If you want to use :meth:`.visualize` or run the test suite, the Graphviz program itself must also be installed; otherwise it is **optional**. +:meth:`ixmp.Reporter.visualize ` uses `Graphviz`_, a program for graph visualization. +Installing ixmp causes the `graphviz `__ Python package to be installed. +If you want to use :meth:`~genno.Computer.visualize` or run the test suite, the Graphviz program itself must also be installed; otherwise it is **optional**. If you install :mod:`ixmp` using Anaconda, Graphviz is installed automatically via `its conda-forge package`_. For other methods of installation, see the `Graphviz download page`_ for downloads and instructions for your system. diff --git a/doc/reporting.rst b/doc/reporting.rst index 2c03b8d46..afaf8385a 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -22,17 +22,19 @@ Top-level classes and functions The following top-level objects from :mod:`genno` may also be imported from :mod:`ixmp.report`. +.. currentmodule:: genno .. autosummary:: - ~genno.core.exceptions.ComputationError - ~genno.core.key.Key - ~genno.core.exceptions.KeyExistsError - ~genno.core.exceptions.MissingKeyError - ~genno.core.quantity.Quantity - ~genno.config.configure + ComputationError + Key + KeyExistsError + MissingKeyError + Quantity + configure :mod:`ixmp.report` additionally defines: +.. currentmodule:: ixmp.report .. autosummary:: Reporter @@ -45,34 +47,37 @@ The following top-level objects from :mod:`genno` may also be imported from Using the :meth:`.from_scenario`, a Reporter is automatically populated with: - - :class:`Keys <.Key>` that retrieve the data for every :mod:`ixmp` item (parameter, variable, equation, or scalar) available in the Scenario. + - :class:`Keys <.genno.Key>` that retrieve the data for every :mod:`ixmp` item (parameter, variable, equation, or scalar) available in the Scenario. .. autosummary:: + finalize from_scenario set_filters The Computer class provides the following methods: + .. currentmodule:: genno .. autosummary:: - ~genno.core.computer.Computer.add - ~genno.core.computer.Computer.add_file - ~genno.core.computer.Computer.add_product - ~genno.core.computer.Computer.add_queue - ~genno.core.computer.Computer.add_single - ~genno.core.computer.Computer.aggregate - ~genno.core.computer.Computer.apply - ~genno.core.computer.Computer.check_keys - ~genno.core.computer.Computer.configure - ~genno.core.computer.Computer.convert_pyam - ~genno.core.computer.Computer.describe - ~genno.core.computer.Computer.disaggregate - ~genno.core.computer.Computer.full_key - ~genno.core.computer.Computer.get - ~genno.core.computer.Computer.infer_keys - ~genno.core.computer.Computer.keys - ~genno.core.computer.Computer.visualize - ~genno.core.computer.Computer.write + + ~Computer.add + ~Computer.add_file + ~Computer.add_product + ~Computer.add_queue + ~Computer.add_single + ~Computer.aggregate + ~Computer.apply + ~Computer.check_keys + ~Computer.configure + ~Computer.convert_pyam + ~Computer.describe + ~Computer.disaggregate + ~Computer.full_key + ~Computer.get + ~Computer.infer_keys + ~Computer.keys + ~Computer.visualize + ~Computer.write .. _reporting-config: @@ -81,7 +86,7 @@ Configuration :mod:`ixmp.report` adds handlers for two configuration sections, and modifies the behaviour of one from :mod:`genno` -.. automethod:: ixmp.report.filters +.. autofunction:: ixmp.report.filters Reporter-specific configuration. @@ -93,7 +98,7 @@ Configuration Keys are dimension IDs. Values are either lists of allowable labels along the respective dimension or :obj:`None` to clear any existing filters for that dimension. - This configuration can be applied through :meth:`.Reporter.set_filters`; :meth:`.Reporter.configure`, or in a configuration file: + This configuration can be applied through :meth:`.Reporter.set_filters`; :meth:`Reporter.configure `, or in a configuration file: .. code-block:: yaml @@ -131,6 +136,7 @@ Operators :mod:`ixmp.report` defines these operators: .. autosummary:: + data_for_quantity map_as_qty update_scenario @@ -139,6 +145,7 @@ Operators Basic operators are defined by :mod:`genno.operator` and its compatibility modules; see there for details: .. autosummary:: + ~genno.compat.plotnine.Plot ~genno.operator.add ~genno.operator.aggregate diff --git a/ixmp/_config.py b/ixmp/_config.py index e55551032..ba349fd77 100644 --- a/ixmp/_config.py +++ b/ixmp/_config.py @@ -29,9 +29,10 @@ def _iter_config_paths(): pass try: - yield "environment (XDG_DATA_HOME)", Path( - os.environ["XDG_DATA_HOME"], "ixmp" - ).resolve() + yield ( + "environment (XDG_DATA_HOME)", + Path(os.environ["XDG_DATA_HOME"], "ixmp").resolve(), + ) except KeyError: pass @@ -273,7 +274,7 @@ def register(self, name: str, type_: type, default: Any = None, **kwargs): Name of the new key. type_ : object Type of valid values for the key, e.g. :obj:`str` or :class:`pathlib.Path`. - default : any, optional + default : optional Default value for the key. If not supplied, the `type` is called to supply the default value, e.g. ``str()``. diff --git a/ixmp/backend/__init__.py b/ixmp/backend/__init__.py index 3f3f24e06..dcfe605a2 100644 --- a/ixmp/backend/__init__.py +++ b/ixmp/backend/__init__.py @@ -5,7 +5,7 @@ #: Lists of field names for tuples returned by Backend API methods. #: #: The key "write_file" refers to the columns appearing in the CSV output from -#: :meth:`export_timeseries_data` when using :class:`.JDBCBackend`. +#: :meth:`.export_timeseries_data` when using :class:`.JDBCBackend`. #: #: .. todo:: Make this consistent with other dimension orders and with :data:`IAMC_IDX`. FIELDS = { @@ -42,7 +42,9 @@ ), } -#: Partial dimensions for “IAMC format”. +#: Partial list of dimensions for the IAMC data structure, or “IAMC format”. This omits +#: "year" and "subannual" which appear in some variants of the structure, but not in +#: others. IAMC_IDX: List[Union[str, int]] = ["model", "scenario", "region", "variable", "unit"] diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index ce399a366..9c10c9a06 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -10,6 +10,7 @@ Hashable, Iterable, List, + Literal, MutableMapping, Optional, Sequence, @@ -60,14 +61,14 @@ def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> Dict[str, Any] """OPTIONAL: Handle platform/backend config arguments. Returns a :class:`dict` to be stored in the configuration file. This - :class:`dict` **must** be valid as keyword arguments to the :meth:`__init__` of - a backend subclass. + :class:`dict` **must** be valid as keyword arguments to the :py:`__init__()` + method of a Backend subclass. The default implementation expects both `args` and `kwargs` to be empty. See also -------- - Config.add_platform + .Config.add_platform """ msg = "Unhandled {} args to Backend.handle_config(): {!r}" if len(args): @@ -83,7 +84,8 @@ def set_log_level(self, level: int) -> None: Parameters ---------- - level : int or Python logging level + level : int + A Python :mod:`logging` level. See also -------- @@ -116,7 +118,7 @@ def set_doc(self, domain: str, docs) -> None: ---------- domain : str Documentation domain, e.g. model, scenario, etc. - docs : dict or array of tuples + docs : dict or iterable of tuple Dictionary or tuple array containing mapping between name of domain object (e.g. model name) and string representing fragment of documentation. """ @@ -409,7 +411,7 @@ def read_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: .xlsx Microsoft Office Open XML spreadsheet ====== === - item_type : ItemType + item_type : .ItemType Type(s) of items to read. Raises @@ -449,7 +451,7 @@ def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: ---------- path : os.PathLike File for output. The filename suffix determines the output format. - item_type : ItemType + item_type : .ItemType Type(s) of items to write. Raises @@ -481,9 +483,9 @@ def write_file(self, path: PathLike, item_type: ItemType, **kwargs) -> None: def init(self, ts: TimeSeries, annotation: str) -> None: """Create a new TimeSeries (or Scenario) `ts`. - init **may** modify the :attr:`~TimeSeries.version` attribute of `ts`. + init **may** modify the :attr:`~.TimeSeries.version` attribute of `ts`. - If `ts` is a :class:`Scenario`; the Backend **must** store the + If `ts` is a :class:`.Scenario`; the Backend **must** store the :attr:`.Scenario.scheme` attribute. Parameters @@ -552,7 +554,7 @@ def commit(self, ts: TimeSeries, comment: str) -> None: @abstractmethod def discard_changes(self, ts: TimeSeries) -> None: - """Discard changes to `ts` since the last :meth:`ts_check_out`.""" + """Discard changes to `ts` since the last :meth:`check_out`.""" @abstractmethod def set_as_default(self, ts: TimeSeries) -> None: @@ -590,7 +592,7 @@ def _handle_rw_filters(filters: dict) -> Tuple[Optional[TimeSeries], Dict]: """Helper for :meth:`read_file` and :meth:`write_file`. The `filters` argument is unpacked if the 'scenarios' key is a single - :class:`TimeSeries` object. A 2-tuple is returned of the object (or + :class:`.TimeSeries` object. A 2-tuple is returned of the object (or :obj:`None`) and the remaining filters. """ ts = None @@ -688,8 +690,8 @@ def set_data( Name of time slice. unit : str Unit symbol. - data : dict (int -> float) - Mapping from year to value. + data : + Mapping from year (:class:`int`) to value (:class:`float`). meta : bool :obj:`True` to mark `data` as metadata. """ @@ -811,8 +813,10 @@ def clone( Returns ------- - Same class as `s` - The cloned Scenario. + Scenario + The cloned Scenario. If `s` is an instance of a subclass of + :class:`ixmp.Scenario`, the returned object **must** be of the same + subclass. """ @abstractmethod @@ -888,22 +892,27 @@ def item_index(self, s: Scenario, name: str, sets_or_names: str) -> List[str]: @abstractmethod def item_get_elements( - self, s: Scenario, type: str, name: str, filters: Dict[str, List[Any]] = None + self, + s: Scenario, + type: Literal["equ", "par", "set", "var"], + name: str, + filters: Dict[str, List[Any]] = None, ) -> Union[Dict[str, Any], pd.Series, pd.DataFrame]: """Return elements of item `name`. Parameters ---------- - type : 'equ' or 'par' or 'set' or 'var' + type : str + Type of the item. name : str Name of the item. - filters : dict (str -> list), optional - If provided, a mapping from dimension names to allowed values along that - dimension. + filters : dict, optional + If provided, a mapping from dimension names (class:`str`) to allowed values + along that dimension (:class:`list`). item_get_elements **must** silently accept values that are *not* members of the set indexing a dimension. Elements which are not :class:`str` **must** - be handled as equivalent to their string representation; i.e. + be handled as equivalent to their string representation; that is, item_get_elements must return the same data for ``filters={'foo': [42]}`` and ``filters={'foo': ['42']}``. @@ -941,7 +950,7 @@ def item_set_elements( type : 'par' or 'set' name : str Name of the items. - elements : iterable of 4-tuple + elements : iterable of tuple The members of each tuple are: ======= ========================== === @@ -967,8 +976,8 @@ def item_set_elements( See also -------- - s_init_item - s_item_delete_elements + init_item + item_delete_elements """ @abstractmethod @@ -987,8 +996,8 @@ def item_delete_elements(self, s: Scenario, type: str, name: str, keys) -> None: See also -------- - s_init_item - s_item_set_elements + init_item + item_set_elements """ @abstractmethod @@ -1015,19 +1024,20 @@ def get_meta( Parameters ---------- - model : str or None + model : str, optional Model name of metadata target. - scenario : str or None + scenario : str, optional Scenario name of metadata target. - version : int or None + version : int, optional :attr:`.TimeSeries.version` of metadata target. strict : bool Only retrieve metadata from the specified target. Returns ------- - dict (str -> any) - Mapping from metadata names/identifiers to values. + dict + Mapping from metadata names/identifiers (:class:`str`) to values + (:class:`Any`). Raises ------ diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index bca14605d..d862cbc42 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -19,8 +19,8 @@ def ts_read_file(ts, path, firstyear=None, lastyear=None): See also -------- - TimeSeries.add_timeseries - TimeSeries.read_file + .TimeSeries.add_timeseries + .TimeSeries.read_file """ if path.suffix == ".csv": @@ -44,7 +44,7 @@ def s_write_excel(be, s, path, item_type, filters=None, max_row=None): See also -------- - Scenario.to_excel + .Scenario.to_excel """ # Default: empty dict filters = filters or dict() @@ -180,7 +180,7 @@ def s_read_excel( # noqa: C901 See also -------- - Scenario.read_excel + .Scenario.read_excel """ log.info(f"Read data from {path}") diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 6001ce1f3..aba605ab7 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -174,11 +174,14 @@ def _wrap(value): class JDBCBackend(CachingBackend): """Backend using JPype/JDBC to connect to Oracle and HyperSQL databases. + This backend is based on the third-party `JPype `_ + Python package that allows interaction with Java code. + Parameters ---------- driver : 'oracle' or 'hsqldb' JDBC driver to use. - path : path-like, optional + path : os.PathLike, optional Path to the HyperSQL database. url : str, optional Partial or complete JDBC URL for the Oracle or HyperSQL database, e.g. @@ -191,8 +194,8 @@ class JDBCBackend(CachingBackend): If :obj:`True` (the default), cache Python objects after conversion from Java objects. jvmargs : str, optional - Java Virtual Machine arguments. See :meth:`.start_jvm`. - dbprops : path-like, optional + Java Virtual Machine arguments. See :func:`.start_jvm`. + dbprops : os.PathLike, optional With ``driver='oracle'``, the path to a database properties file containing `driver`, `url`, `user`, and `password` information. """ @@ -555,7 +558,7 @@ def write_file(self, path, item_type: ItemType, **kwargs): - scenario : str - variable : list of str - default : bool. If :obj:`True`, only data from TimeSeries - versions with :meth:`set_default` are written. + versions with :meth:`.TimeSeries.set_as_default` are written. See also -------- @@ -1171,7 +1174,7 @@ def _get_item(self, s, ix_type, name, load=True): def start_jvm(jvmargs=None): - """Start the Java Virtual Machine via :mod:`JPype`. + """Start the Java Virtual Machine via JPype_. Parameters ---------- diff --git a/ixmp/cli.py b/ixmp/cli.py index 33d759190..40a4478e1 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -14,6 +14,7 @@ class VersionType(click.ParamType): name = "version" # https://github.com/pallets/click/issues/411 def convert(self, value, param, ctx): + """Fail if `value` is not :class:`int` or 'all'.""" if value == "new": return value elif isinstance(value, int): diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 9f7a8b86f..72318fec8 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -25,9 +25,9 @@ class Platform: 2. One or more **model(s)**; codes in Python or other languages or frameworks that run, via :meth:`Scenario.solve`, on the data stored in the Platform. - The Platform parameters control these components. :class:`TimeSeries` and - :class:`Scenario` objects tied to a single Platform; to move data between platforms, - see :meth:`Scenario.clone`. + The Platform parameters control these components. :class:`.TimeSeries` and + :class:`.Scenario` objects tied to a single Platform; to move data between + platforms, see :meth:`Scenario.clone`. Parameters ---------- @@ -166,7 +166,7 @@ def scenario_list( Scenario information, with the columns: - ``model``, ``scenario``, ``version``, and ``scheme``—Scenario identifiers; - see :class:`.Timeseries and :class:`.Scenario`. + see :class:`.TimeSeries` and :class:`.Scenario`. - ``is_default``—:obj:`True` if the ``version`` is the default version for the (``model``, ``scenario``). - ``is_locked``—:obj:`True` if the Scenario has been locked for use. @@ -226,7 +226,7 @@ def export_timeseries_data( Only return data for unit name(s) in this list. region: list of str, optional Only return data for region(s) in this list. - export_all_runs: boolean, optional + export_all_runs: bool, optional Export all existing model+scenario run combinations. """ if export_all_runs and (model or scenario): @@ -311,10 +311,10 @@ def add_region(self, region: str, hierarchy: str, parent: str = "World") -> None ---------- region : str Name of the region. - parent : str, default 'World' - Assign a 'parent' region. hierarchy : str Hierarchy level of the region (e.g., country, R11, basin) + parent : str, optional + Assign a 'parent' region. """ if not self._existing_node(region): self._backend.set_node(region, parent, hierarchy) diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 9e3009d49..a60cdf809 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -31,7 +31,7 @@ class Scenario(TimeSeries): class in :data:`.MODELS` is used to initialize items in the Scenario. cache: .. deprecated:: 3.0 - The `cache` keyword argument to :class:`Scenario` has no effect and raises a + The `cache` keyword argument to :class:`.Scenario` has no effect and raises a warning. Use `cache` as one of the `backend_args` to :class:`Platform` to disable/enable caching for storage backends that support it. Use :meth:`load_scenario_data` to load all data in the Scenario into an in-memory @@ -97,7 +97,7 @@ def check_out(self, timeseries_only: bool = False) -> None: See Also -------- TimeSeries.check_out - utils.maybe_check_out + util.maybe_check_out """ if not timeseries_only and self.has_solution(): raise ValueError( @@ -376,9 +376,9 @@ def par( ---------- name : str Name of the parameter - filters : dict (str -> list of str), optional - Index names mapped to lists of index set elements. Elements not appearing - in the respective index set(s) are silently ignored. + filters : dict, optional + Keys are index names. Values are lists of index set elements. Elements not + appearing in the respective index set(s) are silently ignored. """ if len(kwargs): warn( @@ -394,17 +394,17 @@ def items( Parameters ---------- - type : ItemType, optional - Types of items to iterate, e.g. :data:`ItemType.PAR` for parameters, the - only value currently supported. + type : .ItemType, optional + Types of items to iterate, for instance :attr:`.ItemType.PAR` for + parameters, the only value currently supported. filters : dict, optional Filters for values along dimensions; same as the `filters` argument to :meth:`par`. Yields ------ - (str, object) - Tuples of item name and data. + tuple + Each tuple consists of (item name, item data). """ if type != ItemType.PAR: raise NotImplementedError( @@ -443,10 +443,9 @@ def add_par( ---------- name : str Name of the parameter. - key_or_data : str or iterable of str or range or dict or - :class:`pandas.DataFrame` + key_or_data : str or iterable of str or range or dict or pandas.DataFrame Element(s) to be added. - value : numeric or iterable of numeric, optional + value : float or iterable of float, optional Values. unit : str or iterable of str, optional Unit symbols. @@ -545,7 +544,7 @@ def init_scalar(self, name: str, val: Real, unit: str, comment=None) -> None: ---------- name : str Name of the scalar - val : number + val : float or int Initial value of the scalar. unit : str Unit of the scalar. @@ -565,7 +564,8 @@ def scalar(self, name: str) -> Dict[str, Union[Real, str]]: Returns ------- - {'value': value, 'unit': unit} + dict + with the keys "value" and "unit". """ return self._backend("item_get_elements", "par", name, None) @@ -578,7 +578,7 @@ def change_scalar( ---------- name : str Name of the scalar. - val : number + val : float or int New value of the scalar. unit : str New unit of the scalar. @@ -596,8 +596,11 @@ def remove_par(self, name: str, key=None) -> None: ---------- name : str Name of the parameter. - key : dataframe or key list or concatenated string, optional - Elements to be removed + key : pandas.DataFrame or list or str, optional + Elements to be removed. If a :class:`pandas.DataFrame`, must contain the + same columns (indices/dimensions) as the parameter. If a :class:`list`, a + single key for a single data point; the individual elements must correspond + to the indices/dimensions of the parameter. """ if key is None: self._backend("delete_item", "par", name) @@ -708,8 +711,8 @@ def clone( annotation : str, optional Explanatory comment for the clone commit message to the database. keep_solution : bool, optional - If :py:const:`True`, include all timeseries data and the solution (vars and - equs) from the source scenario in the clone. If :py:const:`False`, only + If :obj:`True`, include all timeseries data and the solution (vars and + equs) from the source scenario in the clone. If :obj:`False`, only include timeseries data marked `meta=True` (see :meth:`.add_timeseries`). shift_first_model_year: int, optional If given, all timeseries data in the Scenario is omitted from the clone for @@ -873,7 +876,7 @@ def to_excel( ---------- path : os.PathLike File to write. Must have suffix :file:`.xlsx`. - items : ItemType, optional + items : .ItemType, optional Types of items to write. Either :attr:`.SET` | :attr:`.PAR` (i.e. only sets and parameters), or :attr:`.MODEL` (also variables and equations, i.e. model solution data). diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index b0fe362d7..c00995258 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -2,7 +2,7 @@ from contextlib import contextmanager, nullcontext from os import PathLike from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Any, Dict, Literal, Optional, Sequence, Tuple, Union from warnings import warn from weakref import ProxyType, proxy @@ -25,7 +25,7 @@ class TimeSeries: """Collection of data in time series format. - TimeSeries is the parent/super-class of :class:`Scenario`. + TimeSeries is the parent/super-class of :class:`.Scenario`. Parameters ---------- @@ -116,7 +116,7 @@ def __del__(self): @classmethod def from_url( - cls, url: str, errors="warn" + cls, url: str, errors: Literal["warn", "raise"] = "warn" ) -> Tuple[Optional["TimeSeries"], Platform]: """Instantiate a TimeSeries (or Scenario) given an ``ixmp://`` URL. @@ -141,8 +141,12 @@ def from_url( Returns ------- - ts, platform : 2-tuple of (TimeSeries, :class:`Platform`) - The TimeSeries and Platform referred to by the URL. + tuple + with 2 elements: + + - The :class:`.TimeSeries` referenced by the `url`. + - The :class:`.Platform` referenced by the `url`, on which the first element + is stored. """ assert errors in ("warn", "raise"), "errors= must be 'warn' or 'raise'" @@ -176,7 +180,7 @@ def check_out(self, timeseries_only: bool = False) -> None: See Also -------- - utils.maybe_check_out + util.maybe_check_out """ self._backend("check_out", timeseries_only) @@ -194,7 +198,7 @@ def commit(self, comment: str) -> None: See Also -------- - utils.maybe_commit + util.maybe_commit """ self._backend("commit", comment) @@ -305,7 +309,7 @@ def add_timeseries( Parameters ---------- - df : :class:`pandas.DataFrame` + df : pandas.DataFrame Data to add. `df` must have the following columns: - `region` or `node` @@ -319,9 +323,9 @@ def add_timeseries( To support subannual temporal resolution of timeseries data, a column `subannual` is optional in `df`. The entries in this column must have been - defined in the Platform instance using :meth:`add_timeslice` beforehand. If + defined in the Platform instance using :meth:`.add_timeslice` beforehand. If no column `subannual` is included in `df`, the data is assumed to contain - yearly values. See :meth:`timeslices` for a detailed description of the + yearly values. See :meth:`.timeslices` for a detailed description of the feature. meta : bool, optional @@ -329,8 +333,8 @@ def add_timeseries( :meth:`Scenario.clone` is called for Scenarios created with ``scheme='MESSAGE'``. - year_lim : tuple of (int or None, int or None), optional - Respectively, minimum and maximum years to add from `df`; data for other + year_lim : tuple, optional + Respectively, earliest and latest years to add from `df`; data for other years is ignored. """ meta = bool(meta) @@ -567,7 +571,7 @@ def set_meta(self, name_or_dict: Union[str, Dict[str, Any]], value=None) -> None name_or_dict : str or dict If :class:`dict`, a mapping of names/identifiers to values. Otherwise, use the metadata identifier. - value : str or number or bool, optional + value : str or float or int or bool, optional Metadata value. """ if isinstance(name_or_dict, str): diff --git a/ixmp/model/base.py b/ixmp/model/base.py index c92da01f3..2f7587693 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -2,9 +2,13 @@ import os import re from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Dict, Mapping from ixmp.util import maybe_check_out, maybe_commit +if TYPE_CHECKING: + from ixmp.core.scenario import Scenario + log = logging.getLogger(__name__) @@ -32,7 +36,7 @@ def __init__(self, name, **kwargs): @classmethod def clean_path(cls, value: str) -> str: - """Subtitute invalid characters in `value` with "_".""" + """Substitute invalid characters in `value` with "_".""" chars = r'<>"/\|?*' + (":" if os.name == "nt" else "") return re.sub("[{}]+".format(re.escape(chars)), "_", value) @@ -53,7 +57,7 @@ def enforce(scenario): Parameters ---------- - scenario : .Scenario + scenario : Scenario Object on which to enforce data consistency. """ @@ -71,7 +75,7 @@ def initialize(cls, scenario): Parameters ---------- - scenario : .Scenario + scenario : Scenario Object to initialize. See also @@ -81,7 +85,7 @@ def initialize(cls, scenario): log.debug(f"No initialization for {repr(scenario.scheme)}-scheme Scenario") @classmethod - def initialize_items(cls, scenario, items): + def initialize_items(cls, scenario: "Scenario", items: Mapping[str, Dict]) -> None: """Helper for :meth:`initialize`. All of the `items` are added to `scenario`. Existing items are not modified. @@ -93,13 +97,13 @@ def initialize_items(cls, scenario, items): Parameters ---------- - scenario : .Scenario + scenario : Scenario Object to initialize. - items : dict of (str -> dict) - Each key is the name of an ixmp item (set, parameter, equation, or - variable) to initialize. Each dict **must** have the key 'ix_type'; one of - 'set', 'par', 'equ', or 'var'; any other entries are keyword arguments to - the methods :meth:`.init_set` etc. + items : + Keys are names of ixmp items (set, parameter, equation, or variable) to + initialize. Values are :class:`dict`, and each **must** have the key + 'ix_type' (one of 'set', 'par', 'equ', or 'var'); any other entries are + keyword arguments to the corresponding methods such as :meth:`.init_set`. Raises ------ @@ -191,6 +195,6 @@ def run(self, scenario): Parameters ---------- - scenario : .Scenario + scenario : Scenario Scenario object to solve by running the Model. """ diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index d1255cc32..47e9aba66 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -62,7 +62,7 @@ class DantzigModel(GAMSModel): - """Dantzig's cannery/transport problem as a :class:`GAMSModel`. + """Dantzig's cannery/transport problem as a :class:`.GAMSModel`. Provided for testing :mod:`ixmp` code. """ diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index c714bd704..963d470c0 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -1,41 +1,52 @@ import logging from itertools import zip_longest +from typing import TYPE_CHECKING, Literal, Mapping import pandas as pd import pint -from genno.core.quantity import Quantity +from genno import Quantity from genno.util import parse_units from ixmp.util import to_iamc_layout from .util import RENAME_DIMS, dims_for_qty, get_reversed_rename_dims +if TYPE_CHECKING: + from ixmp.core.scenario import Scenario + log = logging.getLogger(__name__) -def data_for_quantity(ix_type, name, column, scenario, config): - """Retrieve data from *scenario*. +def data_for_quantity( + ix_type: Literal["equ", "par", "var"], + name: str, + column: Literal["mrg", "lvl", "value"], + scenario: "Scenario", + config: Mapping[str, Mapping], +) -> Quantity: + """Retrieve data from `scenario`. Parameters ---------- - ix_type : 'equ' or 'par' or 'var' + ix_type : Type of the ixmp object. name : str Name of the ixmp object. - column : 'mrg' or 'lvl' or 'value' + column : Data to retrieve. 'mrg' and 'lvl' are valid only for ``ix_type='equ'``,and 'level' otherwise. scenario : ixmp.Scenario Scenario containing data to be retrieved. - config : dict of (str -> dict) - The key 'filters' may contain a mapping from dimensions to iterables of allowed - values along each dimension. The key 'units'/'apply' may contain units to apply - to the quantity; any such units overwrite existing units, without conversion. + config : + Configuration. The key 'filters' may contain a mapping from dimensions to + iterables of allowed values along each dimension. The key 'units'/'apply' may + contain units to apply to the quantity; any such units overwrite existing units, + without conversion. Returns ------- - :class:`Quantity` - Data for *name*. + ~genno.Quantity + Data for `name`. """ log.debug(f"{name}: retrieve data") @@ -123,7 +134,8 @@ def data_for_quantity(ix_type, name, column, scenario, config): try: # Remove length-1 dimensions for scalars - qty = qty.squeeze("index", drop=True) + # TODO Remove exclusion when genno provides a signature for Quantity.squeeze + qty = qty.squeeze("index", drop=True) # type: ignore [attr-defined] except (KeyError, ValueError): # KeyError if "index" does not exist; ValueError if its length is > 1 pass @@ -132,7 +144,7 @@ def data_for_quantity(ix_type, name, column, scenario, config): def map_as_qty(set_df: pd.DataFrame, full_set): - """Convert *set_df* to a :class:`.Quantity`. + """Convert *set_df* to a :class:`~.genno.Quantity`. For the MESSAGE sets named ``cat_*`` (see :ref:`message_ix:mapping-sets`) :meth:`ixmp.Scenario.set` returns a :class:`~pandas.DataFrame` with two columns: @@ -148,7 +160,7 @@ def map_as_qty(set_df: pd.DataFrame, full_set): See also -------- - .broadcast_map + ~genno.operator.broadcast_map """ set_from, set_to = set_df.columns names = [RENAME_DIMS.get(c, c) for c in set_df.columns] @@ -180,7 +192,7 @@ def store_ts(scenario, *data, strict: bool = False) -> None: Scenario on which to store data. data : pandas.DataFrame or pyam.IamDataFrame 1 or more objects containing data to store. If :class:`pandas.DataFrame`, the - data are passed through :func:`to_iamc_layout`. + data are passed through :func:`.to_iamc_layout`. strict: bool If :data:`True` (default :data:`False`), raise an exception if any of `data` are not successfully added. Otherwise, log on level :ref:`ERROR ` and @@ -216,8 +228,8 @@ def update_scenario(scenario, *quantities, params=[]): Parameters ---------- - scenario : .Scenario - quantities : .Quantity or pd.DataFrame + scenario : Scenario + quantities : Quantity or pandas.DataFrame If DataFrame, must be valid input to :meth:`.Scenario.add_par`. params : list of str, optional For every element of `quantities` that is a pd.DataFrame, the element of diff --git a/ixmp/report/reporter.py b/ixmp/report/reporter.py index 1019ddc48..670cd40ba 100644 --- a/ixmp/report/reporter.py +++ b/ixmp/report/reporter.py @@ -27,14 +27,14 @@ def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": Parameters ---------- - scenario : .Scenario + scenario : Scenario Scenario to introspect in creating the Reporter. - kwargs : optional - Passed to :meth:`Scenario.configure`. + kwargs : + Passed to :meth:`genno.Computer.configure`. Returns ------- - .Reporter + Reporter A Reporter instance containing: - A 'scenario' key referring to the *scenario* object. @@ -96,7 +96,7 @@ def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": def finalize(self, scenario: Scenario) -> None: """Prepare the Reporter to act on `scenario`. - The :class:`.TimeSeries` (i.e. including :class:`ixmp.Scenario` and + The :class:`.TimeSeries` (thus also :class:`.Scenario` or :class:`message_ix.Scenario`) object `scenario` is stored with the key ``'scenario'``. All subsequent processing will act on data from this Scenario. """ diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 1cd62fed8..6645b02d2 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -69,7 +69,10 @@ "TS_DF", "add_random_model_data", "add_test_data", + "assert_logs", + "create_test_platform", "get_cell_output", + "ixmp_cli", "make_dantzig", "models", "populate_test_platform", @@ -77,6 +80,8 @@ "random_ts_data", "resource_limit", "run_notebook", + "test_mp", + "tmp_env", ] @@ -150,7 +155,7 @@ def test_data_path(): @pytest.fixture(scope="module") def test_mp(request, tmp_env, test_data_path): - """An empty :class:`Platform` connected to a temporary, in-memory database. + """An empty :class:`.Platform` connected to a temporary, in-memory database. This fixture has **module** scope: the same Platform is reused for all tests in a module. diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index dc08f48a8..5d85ce84a 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -20,14 +20,18 @@ models = SCEN _MS: List[Any] = [models["dantzig"]["model"], models["dantzig"]["scenario"]] + +#: Time series data for testing. HIST_DF = pd.DataFrame( [_MS + ["DantzigLand", "GDP", "USD", 850.0, 900.0, 950.0]], columns=IAMC_IDX + [2000, 2005, 2010], ) +#: Time series data for testing. INP_DF = pd.DataFrame( [_MS + ["DantzigLand", "Demand", "cases", 850.0, 900.0]], columns=IAMC_IDX + [2000, 2005], ) +#: Time series data for testing. TS_DF = ( pd.concat([HIST_DF, INP_DF], sort=False) .sort_values(by="variable") @@ -126,6 +130,7 @@ def add_random_model_data(scenario, length): def add_test_data(scen: Scenario): + """Populate `scen` with test data.""" # New sets t_foo = ["foo{}".format(i) for i in (1, 2, 3)] t_bar = ["bar{}".format(i) for i in (4, 5, 6)] @@ -160,7 +165,7 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen Parameters ---------- - mp : .Platform + mp : Platform Platform on which to create the scenario. solve : bool, optional If :obj:`True`. then solve the scenario before returning. Default :obj:`False`. @@ -169,7 +174,7 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen Returns ------- - .Scenario + Scenario See also -------- @@ -225,7 +230,7 @@ def populate_test_platform(platform): - 3 versions of the Dantzig cannery/transport Scenario. - Version 2 is the default. - - All have :obj:`HIST_DF` and :obj:`TS_DF` as time-series data. + - All have :data:`.HIST_DF` and :data:`.TS_DF` as time-series data. - 1 version of a TimeSeries with model name 'Douglas Adams' and scenario name 'Hitchhiker', containing 2 values. @@ -283,9 +288,9 @@ def random_model_data(length): def random_ts_data(length): - """A :class:`pandas.DataFrame` of time series data with *length* rows. + """A :class:`pandas.DataFrame` of time series data with `length` rows. - Suitable for passage to :meth:`TimeSeries.add_timeseries`. + Suitable for passage to :meth:`.TimeSeries.add_timeseries`. """ return pd.DataFrame.from_dict( dict( diff --git a/ixmp/testing/jupyter.py b/ixmp/testing/jupyter.py index 59b0df5b2..6cead889f 100644 --- a/ixmp/testing/jupyter.py +++ b/ixmp/testing/jupyter.py @@ -13,11 +13,11 @@ def run_notebook(nb_path, tmp_path, env=None, **kwargs): Parameters ---------- - nb_path : path-like + nb_path : os.PathLike The notebook file to execute. - tmp_path : path-like + tmp_path : os.PathLike A directory in which to create temporary output. - env : dict-like, optional + env : dict, optional Execution environment for :mod:`nbclient`. Default: :obj:`os.environ`. kwargs : Keyword arguments for :class:`nbclient.NotebookClient`. Defaults are set for: diff --git a/ixmp/testing/resource.py b/ixmp/testing/resource.py index d71e3bfc6..c7e9da121 100644 --- a/ixmp/testing/resource.py +++ b/ixmp/testing/resource.py @@ -1,6 +1,7 @@ """Performance testing.""" import logging from collections import namedtuple +from typing import Any, Optional try: import resource @@ -30,13 +31,13 @@ def format_meminfo(arr, cls=float): # Variables for memory_usage _COUNT = 0 _PREV = np.zeros(6) -_RT = None +_RT: Optional[Any] = None def memory_usage(message="", reset=False): """Profile memory usage from within a test function. - The Python package ``memory_profiler`` and :mod:`jpype` are used to report memory + The Python package ``memory_profiler`` and JPype_ are used to report memory usage. A message is logged at the ``DEBUG`` level, similar to:: DEBUG ixmp.testing:testing.py:527 42 @@ -134,7 +135,7 @@ def memory_usage(message="", reset=False): @pytest.fixture(scope="function") def resource_limit(request): - """A fixture that limits Python :mod:`resources`. + """A fixture that limits Python :mod:`resources `. See the documentation (``pytest --help``) for the ``--resource-limit`` command-line option that selects (1) the specific resource and (2) the level of the limit. diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index d10729e39..fd8e574ef 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -94,7 +94,7 @@ def check_year(y, s): def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: """Compute the difference between Scenarios `a` and `b`. - :func:`diff` combines :func:`pandas.merge` and :meth:`Scenario.items`. Only + :func:`diff` combines :func:`pandas.merge` and :meth:`.Scenario.items`. Only parameters are compared. :func:`~pandas.merge` is called with the arguments ``how="outer", sort=True, suffixes=("_a", "_b"), indicator=True``; the merge is performed on all columns except 'value' or 'unit'. @@ -174,7 +174,7 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: def discard_on_error(ts: "TimeSeries"): """Context manager to discard changes to `ts` and close the DB on any exception. - For :mod:`JDBCBackend`, this can avoid leaving `ts` in a "locked" state in the + For :class:`.JDBCBackend`, this can avoid leaving `ts` in a "locked" state in the database. Examples @@ -218,7 +218,7 @@ def discard_on_error(ts: "TimeSeries"): def maybe_check_out(timeseries, state=None): """Check out `timeseries` depending on `state`. - If `state` is :obj:`None`, then :meth:`check_out` is called. + If `state` is :obj:`None`, then :meth:`.TimeSeries.check_out` is called. Returns ------- @@ -288,7 +288,7 @@ def maybe_convert_scalar(obj) -> pd.DataFrame: Parameters ---------- obj - Any value returned by :meth:`Scenario.par`. For a scalar (0-dimensional) + Any value returned by :meth:`.Scenario.par`. For a scalar (0-dimensional) parameter, this will be :class:`dict`. Returns @@ -323,9 +323,9 @@ def parse_url(url): Returns ------- platform_info : dict - Keyword argument 'name' for the :class:`Platform` constructor. + Keyword argument 'name' for the :class:`.Platform` constructor. scenario_info : dict - Keyword arguments for a :class:`Scenario` on the above platform: + Keyword arguments for a :class:`.Scenario` on the above platform: 'model', 'scenario' and, optionally, 'version'. Raises @@ -458,7 +458,7 @@ def format_scenario_list( scenario name matches are returned. default_only : bool, optional Only return TimeSeries where a default version has been set with - :meth:`TimeSeries.set_as_default`. + :meth:`.TimeSeries.set_as_default`. as_url : bool, optional Format results as ixmp URLs. From 0790b965d3ca4098b8a6e411fc18c493d069dadc Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 14 Nov 2023 14:29:21 +0100 Subject: [PATCH 11/37] Apply hauntsaninja/no_implicit_optional; closes #465 --- ixmp/_config.py | 2 +- ixmp/backend/base.py | 18 ++++++++----- ixmp/backend/jdbc.py | 20 ++++++++++----- ixmp/core/platform.py | 4 +-- ixmp/core/scenario.py | 57 +++++++++++++++++++++++++---------------- ixmp/core/timeseries.py | 12 ++++----- pyproject.toml | 1 - 7 files changed, 69 insertions(+), 45 deletions(-) diff --git a/ixmp/_config.py b/ixmp/_config.py index ba349fd77..e31ad287a 100644 --- a/ixmp/_config.py +++ b/ixmp/_config.py @@ -265,7 +265,7 @@ def keys(self) -> Tuple[str, ...]: """Return the names of all registered configuration keys.""" return self.values.keys() - def register(self, name: str, type_: type, default: Any = None, **kwargs): + def register(self, name: str, type_: type, default: Optional[Any] = None, **kwargs): """Register a new configuration key. Parameters diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index 9c10c9a06..a13dbd946 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -124,7 +124,7 @@ def set_doc(self, domain: str, docs) -> None: """ @abstractmethod - def get_doc(self, domain: str, name: str = None) -> Union[str, Dict]: + def get_doc(self, domain: str, name: Optional[str] = None) -> Union[str, Dict]: """Read documentation from database Parameters @@ -183,7 +183,11 @@ def get_auth(self, user: str, models: Sequence[str], kind: str) -> Dict[str, boo @abstractmethod def set_node( - self, name: str, parent: str = None, hierarchy: str = None, synonym: str = None + self, + name: str, + parent: Optional[str] = None, + hierarchy: Optional[str] = None, + synonym: Optional[str] = None, ) -> None: """Add a node name to the Platform. @@ -791,7 +795,7 @@ def clone( scenario: str, annotation: str, keep_solution: bool, - first_model_year: int = None, + first_model_year: Optional[int] = None, ) -> Scenario: """Clone `s`. @@ -896,7 +900,7 @@ def item_get_elements( s: Scenario, type: Literal["equ", "par", "set", "var"], name: str, - filters: Dict[str, List[Any]] = None, + filters: Optional[Dict[str, List[Any]]] = None, ) -> Union[Dict[str, Any], pd.Series, pd.DataFrame]: """Return elements of item `name`. @@ -1278,9 +1282,9 @@ def cache( def cache_invalidate( self, ts: TimeSeries, - ix_type: str = None, - name: str = None, - filters: Dict = None, + ix_type: Optional[str] = None, + name: Optional[str] = None, + filters: Optional[Dict] = None, ) -> None: """Invalidate cached values. diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index aba605ab7..ff25644c1 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -7,7 +7,7 @@ from copy import copy from pathlib import Path, PurePosixPath from types import SimpleNamespace -from typing import Generator, List, Mapping +from typing import Generator, List, Mapping, Optional from weakref import WeakKeyDictionary import jpype @@ -1083,9 +1083,9 @@ def item_delete_elements(self, s, type, name, keys): def get_meta( self, - model: str = None, - scenario: str = None, - version: int = None, + model: Optional[str] = None, + scenario: Optional[str] = None, + version: Optional[int] = None, strict: bool = False, ) -> dict: self._validate_meta_args(model, scenario, version) @@ -1100,7 +1100,11 @@ def get_meta( return {entry.getKey(): _unwrap(entry.getValue()) for entry in meta.entrySet()} def set_meta( - self, meta: dict, model: str = None, scenario: str = None, version: int = None + self, + meta: dict, + model: Optional[str] = None, + scenario: Optional[str] = None, + version: Optional[int] = None, ) -> None: self._validate_meta_args(model, scenario, version) if version is not None: @@ -1116,7 +1120,11 @@ def set_meta( _raise_jexception(e) def remove_meta( - self, names, model: str = None, scenario: str = None, version: int = None + self, + names, + model: Optional[str] = None, + scenario: Optional[str] = None, + version: Optional[int] = None, ): self._validate_meta_args(model, scenario, version) if version is not None: diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 72318fec8..8218729dd 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -186,8 +186,8 @@ def export_timeseries_data( self, path: PathLike, default: bool = True, - model: str = None, - scenario: str = None, + model: Optional[str] = None, + scenario: Optional[str] = None, variable=None, unit=None, region=None, diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index a60cdf809..739c97669 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -164,7 +164,10 @@ def has_set(self, name: str) -> bool: return name in self.set_list() def init_set( - self, name: str, idx_sets: Sequence[str] = None, idx_names: Sequence[str] = None + self, + name: str, + idx_sets: Optional[Sequence[str]] = None, + idx_names: Optional[Sequence[str]] = None, ) -> None: """Initialize a new set. @@ -189,7 +192,7 @@ def init_set( return self._backend("init_item", "set", name, idx_sets, idx_names) def set( - self, name: str, filters: Dict[str, Sequence[str]] = None, **kwargs + self, name: str, filters: Optional[Dict[str, Sequence[str]]] = None, **kwargs ) -> Union[List[str], pd.DataFrame]: """Return the (filtered) elements of a set. @@ -213,7 +216,7 @@ def add_set( # noqa: C901 self, name: str, key: Union[str, Sequence[str], Dict, pd.DataFrame], - comment: str = None, + comment: Optional[str] = None, ) -> None: """Add elements to an existing set. @@ -321,7 +324,9 @@ def add_set( # noqa: C901 self._backend("item_set_elements", "set", name, elements) def remove_set( - self, name: str, key: Union[str, Sequence[str], Dict, pd.DataFrame] = None + self, + name: str, + key: Optional[Union[str, Sequence[str], Dict, pd.DataFrame]] = None, ) -> None: """Delete set elements or an entire set. @@ -347,7 +352,10 @@ def has_par(self, name: str) -> bool: return name in self.par_list() def init_par( - self, name: str, idx_sets: Sequence[str], idx_names: Sequence[str] = None + self, + name: str, + idx_sets: Sequence[str], + idx_names: Optional[Sequence[str]] = None, ) -> None: """Initialize a new parameter. @@ -365,7 +373,7 @@ def init_par( return self._backend("init_item", "par", name, idx_sets, idx_names) def par( - self, name: str, filters: Dict[str, Sequence[str]] = None, **kwargs + self, name: str, filters: Optional[Dict[str, Sequence[str]]] = None, **kwargs ) -> pd.DataFrame: """Return parameter data. @@ -388,7 +396,9 @@ def par( return self._backend("item_get_elements", "par", name, filters) def items( - self, type: ItemType = ItemType.PAR, filters: Dict[str, Sequence[str]] = None + self, + type: ItemType = ItemType.PAR, + filters: Optional[Dict[str, Sequence[str]]] = None, ) -> Iterable[Tuple[str, Any]]: """Iterate over model data items. @@ -432,10 +442,10 @@ def items( def add_par( self, name: str, - key_or_data: Union[str, Sequence[str], Dict, pd.DataFrame] = None, + key_or_data: Optional[Union[str, Sequence[str], Dict, pd.DataFrame]] = None, value=None, - unit: str = None, - comment: str = None, + unit: Optional[str] = None, + comment: Optional[str] = None, ) -> None: """Set the values of a parameter. @@ -570,7 +580,7 @@ def scalar(self, name: str) -> Dict[str, Union[Real, str]]: return self._backend("item_get_elements", "par", name, None) def change_scalar( - self, name: str, val: Real, unit: str, comment: str = None + self, name: str, val: Real, unit: str, comment: Optional[str] = None ) -> None: """Set the value and unit of a scalar. @@ -616,7 +626,10 @@ def has_var(self, name: str) -> bool: return name in self.var_list() def init_var( - self, name: str, idx_sets: Sequence[str] = None, idx_names: Sequence[str] = None + self, + name: str, + idx_sets: Optional[Sequence[str]] = None, + idx_names: Optional[Sequence[str]] = None, ) -> None: """Initialize a new variable. @@ -683,12 +696,12 @@ def equ(self, name: str, filters=None, **kwargs) -> pd.DataFrame: def clone( self, - model: str = None, - scenario: str = None, - annotation: str = None, + model: Optional[str] = None, + scenario: Optional[str] = None, + annotation: Optional[str] = None, keep_solution: bool = True, - shift_first_model_year: int = None, - platform: Platform = None, + shift_first_model_year: Optional[int] = None, + platform: Optional[Platform] = None, ) -> "Scenario": """Clone the current scenario and return the clone. @@ -740,7 +753,7 @@ def has_solution(self) -> bool: """Return :obj:`True` if the Scenario contains model solution data.""" return self._backend("has_solution") - def remove_solution(self, first_model_year: int = None) -> None: + def remove_solution(self, first_model_year: Optional[int] = None) -> None: """Remove the solution from the scenario. This function removes the solution (variables and equations) and timeseries @@ -765,8 +778,8 @@ def remove_solution(self, first_model_year: int = None) -> None: def solve( self, - model: str = None, - callback: Callable = None, + model: Optional[str] = None, + callback: Optional[Callable] = None, cb_kwargs: Dict[str, Any] = {}, **model_options, ) -> None: @@ -867,8 +880,8 @@ def to_excel( self, path: PathLike, items: ItemType = ItemType.SET | ItemType.PAR, - filters: Dict[str, Union[Sequence[str], "Scenario"]] = None, - max_row: int = None, + filters: Optional[Dict[str, Union[Sequence[str], "Scenario"]]] = None, + max_row: Optional[int] = None, ) -> None: """Write Scenario to a Microsoft Excel file. diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index c00995258..74a1248ba 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -58,7 +58,7 @@ def __init__( model: str, scenario: str, version: Optional[Union[int, str]] = None, - annotation: str = None, + annotation: Optional[str] = None, **kwargs, ): # Check arguments @@ -382,10 +382,10 @@ def add_timeseries( def timeseries( self, - region: Union[str, Sequence[str]] = None, - variable: Union[str, Sequence[str]] = None, - unit: Union[str, Sequence[str]] = None, - year: Union[int, Sequence[int]] = None, + region: Optional[Union[str, Sequence[str]]] = None, + variable: Optional[Union[str, Sequence[str]]] = None, + unit: Optional[Union[str, Sequence[str]]] = None, + year: Optional[Union[int, Sequence[int]]] = None, iamc: bool = False, subannual: Union[bool, str] = "auto", ) -> pd.DataFrame: @@ -547,7 +547,7 @@ def get_geodata(self) -> pd.DataFrame: # Metadata - def get_meta(self, name: str = None): + def get_meta(self, name: Optional[str] = None): """Get :ref:`data-meta` for this object. Metadata with the given `name`, attached to this (:attr:`model` name, diff --git a/pyproject.toml b/pyproject.toml index 7b6bc5fc0..dd8e20e7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,6 @@ omit = ["ixmp/utils/sphinx_linkcode_github.py"] profile = "black" [tool.mypy] -no_implicit_optional = false [[tool.mypy.overrides]] # Packages/modules for which no type hints are available. From d6fcdb4feb84d5aeb86a0bdb9e6700faf1040ca9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 15 Nov 2023 12:23:17 +0100 Subject: [PATCH 12/37] Ignore Java error when adding "" unit with driver=oracle --- ixmp/backend/jdbc.py | 9 ++++++++- ixmp/tests/backend/test_jdbc.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index ff25644c1..7b5a2c732 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -472,7 +472,14 @@ def get_scenarios(self, default, model, scenario): yield data def set_unit(self, name, comment): - self.jobj.addUnitToDB(name, comment) + try: + self.jobj.addUnitToDB(name, comment) + except Exception as e: # pragma: no cover + if "Error assigning an unit-key-id mapping" in str(e) and "" == str(name): + # ixmp_source does not support adding "" with Oracle + log.warning(f"…skip {repr(name)} (ixmp.JDBCBackend with driver=oracle)") + else: + _raise_jexception(e) def get_units(self): return to_pylist(self.jobj.getUnitList()) diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index 6e5ac7466..f5bf6b64a 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -155,6 +155,12 @@ def test_set_data_inf(self, mp): ): ts.add_timeseries(data) # Calls JDBCBackend.set_data + def test_set_unit(self, caplog, be): + be.set_unit("", "comment") + # No warning issued under pytest/driver=hsqldb; the exception only occurs with + # driver=oracle + assert [] == caplog.messages + def test_read_file(self, tmp_path, be): """Cannot read CSV files.""" with pytest.raises(NotImplementedError): From 65013123c467e00c5b60ad554e7236a2ff711145 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 14:09:51 +0100 Subject: [PATCH 13/37] Work around ixmp_source bugs in JDBCBackend.init_items() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ixmp_source insists on particular idx_sets/idx_names for some MsgScenario items, for instance COST_NODAL. But MsgScenario.initializeVar() fails to initialize the item if *any* idx_sets/idx_names are given—even the correct ones. This change allows Scenario.init_var() to accept the correct values. In this case it silently discards matching values. If non-matching values are given, it raises a Python (rather than Java) exception. --- ixmp/backend/jdbc.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 7b5a2c732..266d5298c 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -5,6 +5,7 @@ from collections import ChainMap from collections.abc import Iterable, Sequence from copy import copy +from functools import lru_cache from pathlib import Path, PurePosixPath from types import SimpleNamespace from typing import Generator, List, Mapping, Optional @@ -44,9 +45,10 @@ java = SimpleNamespace() JAVA_CLASSES = [ + "at.ac.iiasa.ixmp.dto.TimesliceDTO", "at.ac.iiasa.ixmp.exceptions.IxException", + "at.ac.iiasa.ixmp.modelspecs.MESSAGEspecs", "at.ac.iiasa.ixmp.objects.Scenario", - "at.ac.iiasa.ixmp.dto.TimesliceDTO", "at.ac.iiasa.ixmp.Platform", "java.lang.Double", "java.lang.Exception", @@ -139,6 +141,19 @@ def _raise_jexception(exc, msg="unhandled Java exception: "): raise RuntimeError(msg) from None +@lru_cache +def _fixed_index_sets(scheme: str) -> Mapping[str, List[str]]: + """Return index sets for items that are fixed in the Java code. + + See :meth:`JDBCBackend.init_item`. The return value is cached so the method is only + called once. + """ + if scheme == "MESSAGE": + return {k: to_pylist(v) for k, v in java.MESSAGEspecs.getIndexDimMap().items()} + else: + return {} + + def _domain_enum(domain): domain_enum = java.DocumentationKey.DocumentationDomain try: @@ -893,12 +908,29 @@ def list_items(self, s, type): return to_pylist(getattr(self.jindex[s], f"get{type.title()}List")()) def init_item(self, s, type, name, idx_sets, idx_names): - # generate index-set and index-name lists + # Generate index-set and index-name lists if isinstance(idx_sets, set) or isinstance(idx_names, set): raise TypeError("index dimension must be string or ordered lists") + # Check `idx_sets` against values hard-coded in ixmp_source + try: + sets = _fixed_index_sets(s.scheme)[name] + except KeyError: + pass + else: + if idx_sets == sets: + # Match → provide empty lists for idx_sets and idx_names. ixmp_source + # raises an exception if any values—even correct ones—are given. + idx_sets = idx_names = [] + else: + raise NotImplementedError( + f"Initialize {type} {name!r} with dimensions {idx_sets} != {sets}" + ) + + # Convert to Java data structure idx_sets = to_jlist(idx_sets) if len(idx_sets) else None + # Handle `idx_names`, if any if idx_names: if len(idx_names) != len(idx_sets): raise ValueError( From 4ad872618b9f04988da39dfe257d19df258c193d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 14:16:41 +0100 Subject: [PATCH 14/37] Test JDBCBackend.init_item() --- ixmp/tests/backend/test_jdbc.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index f5bf6b64a..afd254e94 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -141,6 +141,36 @@ def test_handle_config_path(self, tmp_path, klass): assert dict(driver="hsqldb", path=tmp_path) == klass.handle_config(args, kwargs) + @pytest.mark.parametrize( + "name, idx_sets", + ( + ("C", ["node", "year"]), + ("COST_NODAL", ["node", "year"]), + ("COST_NODAL_NET", ["node", "year"]), + ("DEMAND", ["node", "commodity", "level", "year", "time"]), + ("GDP", ["node", "year"]), + ("I", ["node", "year"]), + # Expected exception class raised on invalid arguments + pytest.param( + "C", + ["node", "year", "technology"], + marks=pytest.mark.xfail(raises=NotImplementedError), + ), + ), + ) + def test_init_item(self, mp, be, name, idx_sets): + """init_item() supports items that have fixed indices in ixmp_source.""" + + # Create an ixmp.Scenario, coerce its scheme to MESSAGE, and then call + # Backend.init(). This creates a Java MsgScenario object by circumventing + # safeguards in Scenario.__init__(). + s = ixmp.Scenario(mp, "model name", "scenario name", version="new") + s.scheme = "MESSAGE" + be.init(s, "") + + # Initialize the item; succeeds + be.init_item(s, type="var", name=name, idx_sets=idx_sets, idx_names=idx_sets) + def test_set_data_inf(self, mp): """:meth:`JDBCBackend.set_data` errors on :data:`numpy.inf` values.""" # Make `mp` think it is connected to an Oracle database From 88de2a4e9c95db132d4102e8585cdc6059ea3f5c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 14:23:10 +0100 Subject: [PATCH 15/37] Support equ, set, var in Scenario.items() - Add Scenario.item_list(), Scenario.has_item(). - Use these to provide Scenario.*_list(), Scenario.has_*(). --- ixmp/core/scenario.py | 119 ++++++++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 739c97669..11dff55c8 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -1,9 +1,10 @@ import logging +from functools import partialmethod from itertools import repeat, zip_longest from numbers import Real from os import PathLike from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Union from warnings import warn import pandas as pd @@ -155,14 +156,6 @@ def _keys(self, name, key_or_keys): else: return [str(key_or_keys)] - def set_list(self) -> List[str]: - """List all defined sets.""" - return self._backend("list_items", "set") - - def has_set(self, name: str) -> bool: - """Check whether the scenario has a set *name*.""" - return name in self.set_list() - def init_set( self, name: str, @@ -343,14 +336,6 @@ def remove_set( else: self._backend("item_delete_elements", "set", name, self._keys(name, key)) - def par_list(self) -> List[str]: - """List all defined parameters.""" - return self._backend("list_items", "par") - - def has_par(self, name: str) -> bool: - """Check whether the scenario has a parameter with that name.""" - return name in self.par_list() - def init_par( self, name: str, @@ -399,14 +384,17 @@ def items( self, type: ItemType = ItemType.PAR, filters: Optional[Dict[str, Sequence[str]]] = None, - ) -> Iterable[Tuple[str, Any]]: + *, + indexed_by: Optional[str] = None, + par_data: bool = True, + ) -> Iterable[str]: """Iterate over model data items. Parameters ---------- type : .ItemType, optional Types of items to iterate, for instance :attr:`.ItemType.PAR` for - parameters, the only value currently supported. + parameters. filters : dict, optional Filters for values along dimensions; same as the `filters` argument to :meth:`par`. @@ -416,28 +404,73 @@ def items( tuple Each tuple consists of (item name, item data). """ - if type != ItemType.PAR: - raise NotImplementedError( - f"Scenario.items(type={type}); only ItemType.PAR is supported" + if filters is None: + filters = dict() + elif type != ItemType.PAR: + log.warning( + "Scenario.items(…, filters=…) has no effect for item type" + + repr(type.name) ) - filters = filters or dict() - - names = sorted(self.par_list()) + names = sorted(self._backend("list_items", str(type.name).lower())) - for name in sorted(names): + for name in names: idx_names = set(self.idx_names(name)) - if len(filters) and not set(filters.keys()) & idx_names: - # No overlap between the filters and this item's dimensions + idx_sets = set(self.idx_sets(name)) + + # Skip if: + # - No overlap between given filters and this item's dimensions; or + # - indexed_by= is given but is not in the index sets of `name`. + if (len(filters) and not set(filters) & idx_names) or ( + indexed_by not in (idx_sets | {None}) + ): continue - # Retrieve the data, reducing the filters to only the dimensions of the item - yield ( - name, - self.par( - name, filters={k: v for k, v in filters.items() if k in idx_names} - ), - ) + if type is ItemType.PAR and par_data: + # Retrieve the data, reducing the filters to only the dimensions of the + # item + _filters = {k: v for k, v in filters.items() if k in idx_names} + yield (name, self.par(name, filters=_filters)) # type: ignore [misc] + else: + yield name + + def has_item(self, name: str, item_type=ItemType.MODEL) -> bool: + """Check whether the Scenario has an item `name` of `item_type`. + + See also + -------- + items + """ + return name in self.items(item_type, par_data=False) + + #: Check whether the scenario has a equation `name`. + has_equ = partialmethod(has_item, item_type=ItemType.EQU) + #: Check whether the scenario has a parameter `name`. + has_par = partialmethod(has_item, item_type=ItemType.PAR) + #: Check whether the scenario has a set `name`. + has_set = partialmethod(has_item, item_type=ItemType.SET) + #: Check whether the scenario has a variable `name`. + has_var = partialmethod(has_item, item_type=ItemType.VAR) + + def list_items( + self, item_type=ItemType.MODEL, indexed_by: Optional[str] = None + ) -> List[str]: + """List all defined items of type `item_type`. + + See also + -------- + items + """ + return list(self.items(item_type, indexed_by=indexed_by, par_data=False)) + + #: List all defined equations. + equ_list = partialmethod(list_items, item_type=ItemType.EQU) + #: List all defined parameters. + par_list = partialmethod(list_items, item_type=ItemType.PAR) + #: List all defined sets. + set_list = partialmethod(list_items, item_type=ItemType.SET) + #: List all defined variables. + var_list = partialmethod(list_items, item_type=ItemType.VAR) def add_par( self, @@ -617,14 +650,6 @@ def remove_par(self, name: str, key=None) -> None: else: self._backend("item_delete_elements", "par", name, self._keys(name, key)) - def var_list(self) -> List[str]: - """List all defined variables.""" - return self._backend("list_items", "var") - - def has_var(self, name: str) -> bool: - """Check whether the scenario has a variable with that name.""" - return name in self.var_list() - def init_var( self, name: str, @@ -658,10 +683,6 @@ def var(self, name: str, filters=None, **kwargs): """ return self._backend("item_get_elements", "var", name, filters) - def equ_list(self) -> List[str]: - """List all defined equations.""" - return self._backend("list_items", "equ") - def init_equ(self, name: str, idx_sets=None, idx_names=None) -> None: """Initialize a new equation. @@ -678,10 +699,6 @@ def init_equ(self, name: str, idx_sets=None, idx_names=None) -> None: idx_names = as_str_list(idx_names) return self._backend("init_item", "equ", name, idx_sets, idx_names) - def has_equ(self, name: str) -> bool: - """Check whether the scenario has an equation with that name.""" - return name in self.equ_list() - def equ(self, name: str, filters=None, **kwargs) -> pd.DataFrame: """Return a dataframe of (filtered) elements for a specific equation. From 18566b6e388e377eacb0aa54518bc3ecba2603f6 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 14:23:31 +0100 Subject: [PATCH 16/37] Expand tests of Scenario.items() --- ixmp/tests/core/test_scenario.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index fdeb41c1b..470c5c027 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -241,7 +241,7 @@ def test_par(self, scen): with pytest.warns(DeprecationWarning, match="ignored kwargs"): scen.par("d", i=["seattle"]) - def test_items(self, scen): + def test_items0(self, scen): # Without filters iterator = scen.items() @@ -270,9 +270,23 @@ def test_items(self, scen): assert i == 1 - with pytest.raises(NotImplementedError): - # NB next() is required here to attempt to generate the first item - next(scen.items(ixmp.ItemType.SET)) + @pytest.mark.parametrize( + "item_type, indexed_by, exp", + ( + (ixmp.ItemType.EQU, None, ["cost", "demand", "supply"]), + (ixmp.ItemType.PAR, None, ["a", "b", "d", "f"]), + (ixmp.ItemType.SET, None, ["i", "j"]), + (ixmp.ItemType.VAR, None, ["x", "z"]), + # With indexed_by= + (ixmp.ItemType.EQU, "i", ["supply"]), + (ixmp.ItemType.PAR, "i", ["a", "d"]), + (ixmp.ItemType.SET, "i", []), + (ixmp.ItemType.VAR, "i", ["x"]), + ), + ) + def test_items1(self, scen, item_type, indexed_by, exp): + # Function runs and yields the expected sequence of item names + assert exp == list(scen.items(item_type, indexed_by=indexed_by, par_data=False)) def test_var(self, scen): df = scen.var("x", filters={"i": ["seattle"]}) From 5ed44d9395fa1d56703605eb3eac668f67210fc5 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 15:19:54 +0100 Subject: [PATCH 17/37] Add "ixmp platform copy" CLI command Lightly modified from message_data.model.transport.cli. --- ixmp/cli.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/ixmp/cli.py b/ixmp/cli.py index 40a4478e1..99b828e06 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -325,6 +325,71 @@ def list_platforms(): print(f"{key}: {info}") +@platform_group.command("copy") +@click.option("--go", is_flag=True, help="Actually manipulate files.") +@click.argument("name_source", metavar="SRC") +@click.argument("name_dest", metavar="DEST") +def copy_platform(go, name_source, name_dest): + """Create the local JDBCBackend/HyperSQL platform DEST as a copy of SRC. + + Any existing data at DEST are overwritten. Without --go, no action occurs. + """ + import shutil + from copy import deepcopy + + def _check(name): + """Retrieve platform configuration and check.""" + _, cfg = ixmp.config.get_platform_info(name) + + # Check that the source platform is supported + info = (cfg["class"], cfg["driver"]) + if info != ("jdbc", "hsqldb"): + msg = f"platform {name!r} has class/driver {info} != ('jdbc', 'hsqldb')" + raise click.ClickException(msg) + + return cfg + + # Retrieve configuration for the source platform + cfg_source = _check(name_source) + + try: + # Retrieve configuration for the destination platform + cfg_dest = _check(name_dest) + add_platform = False + except ValueError: + # Target platform does not exist; construct its configuration + cfg_dest = deepcopy(cfg_source) + cfg_dest["path"] = Path(cfg_dest["path"]).parent.joinpath(name_dest) + add_platform = True + + # Base paths for file operations + path_source = Path(cfg_source["path"]) + dir_dest = Path(cfg_dest["path"]).parent + + msg = "" if go else "(dry run) " + + # Iterate over all files with `path_source` as a base name; skip .log and + # .properties files + for path in filter( + lambda p: p.suffix not in {".log", ".properies"}, + path_source.parent.glob(f"{path_source.stem}.*"), + ): + # Destination path + path_dest = dir_dest.joinpath(name_dest).with_suffix(path.suffix) + + print(f"{msg}Copy {path} → {path_dest}") + if not go and path_dest.exists(): + print(f"{' ' * len(msg)}(would replace existing file)") + + if go: + shutil.copyfile(path, path_dest) + + if go and add_platform: + # Store configuration for newly-created platform + ixmp.config.add_platform(name_dest, "jdbc", "hsqldb", cfg_dest["path"]) + ixmp.config.save() + + @main.command("list") @click.option( "--match", From 5d4c0aa408534f3cea0a58f11e33513ae0dd197f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 15:25:27 +0100 Subject: [PATCH 18/37] Add reporting operators from_url, {get,remove}_ts --- ixmp/report/operator.py | 85 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index 963d470c0..d8329a420 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -1,12 +1,13 @@ import logging from itertools import zip_longest -from typing import TYPE_CHECKING, Literal, Mapping +from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Set, Union import pandas as pd import pint from genno import Quantity from genno.util import parse_units +from ixmp.core.timeseries import TimeSeries from ixmp.util import to_iamc_layout from .util import RENAME_DIMS, dims_for_qty, get_reversed_rename_dims @@ -143,6 +144,46 @@ def data_for_quantity( return qty +# Non-weak references to objects to keep them alive +_FROM_URL_REF: Set[Any] = set() + + +def from_url(url: str, cls=TimeSeries) -> "TimeSeries": + """Return a :class:`.TimeSeries` or subclass instance, given its `url`. + + Parameters + ---------- + cls : type, optional + Subclass to instantiate and return; for instance, :class:`.Scenario`. + """ + ts, mp = cls.from_url(url) + assert ts is not None + _FROM_URL_REF.add(ts) + _FROM_URL_REF.add(mp) + return ts + + +def get_ts( + ts: "TimeSeries", + filters: Optional[dict] = None, + iamc: bool = False, + subannual: Union[bool, str] = "auto", +) -> pd.DataFrame: + """Retrieve timeseries data from `ts`. + + Corresponds to :meth:`.TimeSeries.timeseries`. + + Parameters + ---------- + filters : + Names and values for the `region`, `variable`, `unit`, and `year` keyword + arguments to :meth:`.timeseries`. + """ + filters = filters or dict() + + return ts.timeseries(iamc=iamc, subannual=subannual, **filters) + + def map_as_qty(set_df: pd.DataFrame, full_set): """Convert *set_df* to a :class:`~.genno.Quantity`. @@ -180,6 +221,48 @@ def map_as_qty(set_df: pd.DataFrame, full_set): ) +def remove_ts( + ts: "TimeSeries", + data: Optional[pd.DataFrame] = None, + after: Optional[int] = None, +) -> None: + """Remove all time series data from `ts`. + + Note that data stored with :meth:`.add_timeseries` using :py:`meta=True` as a + keyword argument cannot be removed using :meth:`.TimeSeries.remove_timeseries`, and + thus also not with this operator. + + Parameters + ---------- + data : pandas.DataFrame, optional + Specific data to be removed. If not given, all time series data is removed. + after : int, optional + If given, only data with `year` labels equal to or greater than `after` are + removed. + """ + if data is None: + data = ts.timeseries().drop("value", axis=1) + + N = len(data) + count = f"{N}" + + if after: + query = f"{after} <= year" + data = data.query(query) + count = f"{len(data)} of {N} ({query})" + + log.info(f"Remove {count} rows of time series data from {ts.url}") + + # TODO improve TimeSeries.transact() to allow timeseries_only=True; use here + ts.check_out(timeseries_only=True) + try: + ts.remove_timeseries(data) + except Exception: + ts.discard_changes() + else: + ts.commit(f"Remove time series data ({__name__}.remove_ts)") + + def store_ts(scenario, *data, strict: bool = False) -> None: """Store time series `data` on `scenario`. From 045405f629991d30543b0f7e2195aba4958d37a9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 15:43:44 +0100 Subject: [PATCH 19/37] Test from_url, {get,remove}_ts --- ixmp/tests/report/test_operator.py | 65 +++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/ixmp/tests/report/test_operator.py b/ixmp/tests/report/test_operator.py index 08ef68330..5628d0e38 100644 --- a/ixmp/tests/report/test_operator.py +++ b/ixmp/tests/report/test_operator.py @@ -9,15 +9,76 @@ from genno.testing import assert_qty_equal from pandas.testing import assert_frame_equal -from ixmp import Scenario +from ixmp import Scenario, TimeSeries from ixmp.model.dantzig import DATA as dantzig_data -from ixmp.report.operator import map_as_qty, store_ts, update_scenario +from ixmp.report.operator import ( + from_url, + get_ts, + map_as_qty, + remove_ts, + store_ts, + update_scenario, +) from ixmp.testing import DATA as test_data from ixmp.testing import assert_logs, make_dantzig pytestmark = pytest.mark.usefixtures("parametrize_quantity_class") +def test_from_url(test_mp): + ts = make_dantzig(test_mp) + + full_url = f"ixmp://{ts.platform.name}/{ts.url}" + + # Operator runs + result = from_url(full_url) + # Result is of the default class + assert result.__class__ is TimeSeries + # Same object was retrieved + assert ts.url == result.url + + # Same, but specifying Scenario + result = from_url(full_url, Scenario) + assert result.__class__ is Scenario + assert ts.url == result.url + + +def test_get_remove_ts(caplog, test_mp): + ts = make_dantzig(test_mp) + + caplog.set_level(logging.INFO, "ixmp") + + # get_ts() runs + result0 = get_ts(ts) + assert_frame_equal(ts.timeseries(), result0) + + # Can be used through a Computer + + c = Computer() + c.require_compat("ixmp.report.operator") + c.add("scenario", ts) + + key = c.add("test1", "get_ts", "scenario", filters=dict(variable="GDP")) + result1 = c.get(key) + assert 3 == len(result1) + + # remove_ts() can be used through Computer + key = c.add("test2", "remove_ts", "scenario", after=1964) + + # Task runs, logs + with assert_logs(caplog, "Remove 5 of 5 (1964 <= year) rows of time series data"): + c.get(key) + + # See comment above; only one row is removed + assert 6 - 3 == len(ts.timeseries()) + + # remove_ts() can be used directly + remove_ts(ts) + + # All non-'meta' data were removed + assert 3 == len(ts.timeseries()) + + def test_map_as_qty(): b = ["b1", "b2", "b3", "b4"] input = pd.DataFrame( From dd0cb18f3cf726a1e06a4d9a9112bfc6c40f10f6 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 16:01:07 +0100 Subject: [PATCH 20/37] Test "ixmp platform copy" --- ixmp/cli.py | 4 ++-- ixmp/tests/test_cli.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ixmp/cli.py b/ixmp/cli.py index 99b828e06..0a0169624 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -369,9 +369,9 @@ def _check(name): msg = "" if go else "(dry run) " # Iterate over all files with `path_source` as a base name; skip .log and - # .properties files + # .properties files and .tmp directory for path in filter( - lambda p: p.suffix not in {".log", ".properies"}, + lambda p: p.suffix not in {".log", ".properties", ".tmp"}, path_source.parent.glob(f"{path_source.stem}.*"), ): # Destination path diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index dbd9d81e8..be641955a 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -1,3 +1,4 @@ +import re from pathlib import Path import pandas as pd @@ -163,6 +164,41 @@ def call(*args, exit_0=True): assert UsageError.exit_code == r.exit_code +def test_platform_copy(ixmp_cli, tmp_path): + """Test 'platform' command.""" + + def call(*args, exit_0=True): + result = ixmp_cli.invoke(["platform"] + list(map(str, args))) + assert not exit_0 or result.exit_code == 0, result.output + return result + + # Add some temporary platform configuration + call("add", "p1", "jdbc", "oracle", "HOSTNAME", "USER", "PASSWORD") + call("add", "p2", "jdbc", "hsqldb", tmp_path.joinpath("p2")) + # Force connection to p2 so that files are created + ixmp_cli.invoke(["--platform=p2", "list"]) + + # Dry-run produces expected output + r = call("copy", "p2", "p3") + assert re.search("Copy .*p2.script → .*p3.script", r.output) + with pytest.raises(ValueError): + # New platform configuration is not saved + ixmp.config.get_platform_info("p3") + + # --go actually copies files, saves new platform config + r = call("copy", "--go", "p2", "p3") + assert tmp_path.joinpath("p3.script").exists() + assert ixmp.config.get_platform_info("p3") + + # Dry-run again with existing config and files + r = call("copy", "p2", "p3") + assert "would replace existing file" in r.output + + # Copying a non-HyperSQL-backed platform fails + with pytest.raises(AssertionError): + call("copy", "p1", "p3") + + def test_import_ts(ixmp_cli, test_mp, test_data_path): # Ensure the 'canning problem'/'standard' TimeSeries exists populate_test_platform(test_mp) From 011577ab8fb01bc8b9f240c1efb6946e2880aa18 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 16:04:37 +0100 Subject: [PATCH 21/37] Exclude TYPE_CHECKING blocks from coverage --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index dd8e20e7a..520346414 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,10 @@ tests = [ ixmp = "ixmp.cli:main" [tool.coverage.report] +exclude_also = [ + # Imports only used by type checkers + "if TYPE_CHECKING:", +] omit = ["ixmp/utils/sphinx_linkcode_github.py"] [tool.isort] From 58b0e149bc91bad36451466010afbe451c65f5a7 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 16:05:10 +0100 Subject: [PATCH 22/37] Adjust workaround for #494 Add Python 3.12 to CI matrix; commented pending #501. --- .github/workflows/pytest.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 5ed6064d6..7b8a95dbe 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -28,7 +28,8 @@ jobs: - "3.8" # Earliest version supported by ixmp - "3.9" - "3.10" - - "3.11" # Latest supported by ixmp + - "3.11" # Latest supported by ixmp + # - "3.12" # Pending JPype support; see iiasa/ixmp#501 # commented: force a specific version of pandas, for e.g. pre-release # testing @@ -91,12 +92,13 @@ jobs: run: echo "RETICULATE_PYTHON=$pythonLocation" >> $GITHUB_ENV shell: bash + - name: Work around https://bugs.launchpad.net/lxml/+bug/2035206 + if: matrix.python-version == '3.8' + run: pip install "lxml != 4.9.3" + - name: Install Python package and dependencies # [docs] contains [tests], which contains [report,tutorial] run: | - # Work around https://bugs.launchpad.net/lxml/+bug/2035206 - pip install "lxml != 4.9.3" - pip install .[docs] # commented: use with "pandas-version" in the matrix, above From e6740b79a6a7bf3656c00e65b4496c7935586f82 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 16:29:49 +0100 Subject: [PATCH 23/37] =?UTF-8?q?Rename=20test=5Futils=20=E2=86=92=20test?= =?UTF-8?q?=5Futil?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ixmp/tests/{test_utils.py => test_util.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ixmp/tests/{test_utils.py => test_util.py} (100%) diff --git a/ixmp/tests/test_utils.py b/ixmp/tests/test_util.py similarity index 100% rename from ixmp/tests/test_utils.py rename to ixmp/tests/test_util.py From 5f9b1a952367deb05ce3eabc0f1de7458d3b76ef Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 16 Nov 2023 16:35:21 +0100 Subject: [PATCH 24/37] Test warnings from DeprecatedPathFinder --- ixmp/tests/test_util.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 5e78edcc6..98a7fd1ed 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -11,6 +11,16 @@ from ixmp.testing import make_dantzig, populate_test_platform +class TestDeprecatedPathFinder: + def test_import(self): + with pytest.warns( + DeprecationWarning, + match="Importing from 'ixmp.reporting.computations' is deprecated and will " + "fail in a future version. Use 'ixmp.report.operator'.", + ): + import ixmp.reporting.computations # type: ignore # noqa: F401 + + def test_check_year(): # If y is a string value, raise a Value Error. From b2a9bc8fdf39c7b5de097092a2d0009575fc9c7d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 11:40:05 +0100 Subject: [PATCH 25/37] Import RENAME_DIMS in genno config handler --- ixmp/report/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ixmp/report/__init__.py b/ixmp/report/__init__.py index 58c44625e..e1d7aaa35 100644 --- a/ixmp/report/__init__.py +++ b/ixmp/report/__init__.py @@ -48,7 +48,9 @@ def filters(c: Computer, filters: dict): @genno.config.handles("rename_dims", iterate=False) def rename_dims(c: Computer, info: dict): """Handle the entire ``rename_dims:`` config section.""" - RENAME_DIMS.update(info) + from ixmp.report import util + + util.RENAME_DIMS.update(info) # keep=True is different vs. genno.config From cc0d667182b14c43e55c486d7325a20b01fb75d0 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 11:40:49 +0100 Subject: [PATCH 26/37] Adjust test_reporter.test_configure() Use protect_rename_dims fixture. --- ixmp/tests/report/test_reporter.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index 7685c1395..afd801867 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -7,7 +7,6 @@ import ixmp from ixmp.report.reporter import Reporter -from ixmp.report.util import RENAME_DIMS from ixmp.testing import add_test_data, assert_logs, make_dantzig pytestmark = pytest.mark.usefixtures("parametrize_quantity_class") @@ -23,8 +22,9 @@ def scenario(test_mp): yield scen +@pytest.mark.usefixtures("protect_rename_dims") def test_configure(test_mp, test_data_path): - # Configure globally; reads 'rename_dims' section + # Configure globally; handles 'rename_dims' section configure(rename_dims={"i": "i_renamed"}) # Reporting uses the RENAME_DIMS mapping of 'i' to 'i_renamed' @@ -37,9 +37,6 @@ def test_configure(test_mp, test_data_path): assert "d:i-j" not in rep, rep.graph.keys() pytest.raises(KeyError, rep.get, "i") - # Remove the configuration for renaming 'i', so that other tests work - RENAME_DIMS.pop("i") - def test_reporter_from_scenario(scenario): r = Reporter.from_scenario(scenario) @@ -180,9 +177,9 @@ def test_cli(ixmp_cli, test_mp, test_data_path): seattle chicago 1\.7 new-york 2\.5 topeka 1\.8 -(Name: value, )?dtype: float64(, units: dimensionsless)?""", +(Name: value, )?dtype: float64(, units: dimensionless)?""", result.output, - ) + ), result.output def test_filters(test_mp, tmp_path, caplog): From 6de44bf3786485d3b609ea3611ed62e867197ccf Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 12:41:16 +0100 Subject: [PATCH 27/37] Import RENAME_DIMS from a common submodule - This avoids other modules getting a module-specific global also named RENAME_DIMS, that may be out of sync with the common one. - Preserves behaviour downstream in message_ix and message-ix-models. - Absolute imports ("from ixmp.report import common") MUST be used; a relative "from . import common" does not work. --- ixmp/report/__init__.py | 15 ++++++++++----- ixmp/report/common.py | 5 +++++ ixmp/report/operator.py | 7 ++++--- ixmp/report/reporter.py | 5 +++-- ixmp/report/util.py | 16 ++++++++++------ 5 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 ixmp/report/common.py diff --git a/ixmp/report/__init__.py b/ixmp/report/__init__.py index e1d7aaa35..4842cef10 100644 --- a/ixmp/report/__init__.py +++ b/ixmp/report/__init__.py @@ -9,12 +9,12 @@ configure, ) +from ixmp.report import common + from .reporter import Reporter -from .util import RENAME_DIMS __all__ = [ # ixmp-specific - "RENAME_DIMS", "Reporter", # Re-exports from genno "ComputationError", @@ -48,9 +48,7 @@ def filters(c: Computer, filters: dict): @genno.config.handles("rename_dims", iterate=False) def rename_dims(c: Computer, info: dict): """Handle the entire ``rename_dims:`` config section.""" - from ixmp.report import util - - util.RENAME_DIMS.update(info) + common.RENAME_DIMS.update(info) # keep=True is different vs. genno.config @@ -58,3 +56,10 @@ def rename_dims(c: Computer, info: dict): def units(c: Computer, info: dict): """Handle the entire ``units:`` config section.""" genno.config.units(c, info) + + +def __getattr__(name: str): + if name == "RENAME_DIMS": + return common.RENAME_DIMS + else: + raise AttributeError(name) diff --git a/ixmp/report/common.py b/ixmp/report/common.py new file mode 100644 index 000000000..97f3b0efe --- /dev/null +++ b/ixmp/report/common.py @@ -0,0 +1,5 @@ +from typing import Dict + +#: Dimensions to rename when extracting raw data from Scenario objects. +#: Mapping from Scenario dimension name -> preferred dimension name. +RENAME_DIMS: Dict[str, str] = {} diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index d8329a420..8176a002a 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -8,9 +8,10 @@ from genno.util import parse_units from ixmp.core.timeseries import TimeSeries +from ixmp.report import common from ixmp.util import to_iamc_layout -from .util import RENAME_DIMS, dims_for_qty, get_reversed_rename_dims +from .util import dims_for_qty, get_reversed_rename_dims if TYPE_CHECKING: from ixmp.core.scenario import Scenario @@ -126,7 +127,7 @@ def data_for_quantity( # Set index if 1 or more dimensions if len(dims): # First rename, then set index - data = data.rename(columns=RENAME_DIMS).set_index(dims) + data = data.rename(columns=common.RENAME_DIMS).set_index(dims) # Convert to a Quantity, assign attrbutes and name qty = Quantity( @@ -204,7 +205,7 @@ def map_as_qty(set_df: pd.DataFrame, full_set): ~genno.operator.broadcast_map """ set_from, set_to = set_df.columns - names = [RENAME_DIMS.get(c, c) for c in set_df.columns] + names = [common.RENAME_DIMS.get(c, c) for c in set_df.columns] # Add an 'all' mapping set_df = pd.concat( diff --git a/ixmp/report/reporter.py b/ixmp/report/reporter.py index 670cd40ba..3c7fcfd1e 100644 --- a/ixmp/report/reporter.py +++ b/ixmp/report/reporter.py @@ -6,9 +6,10 @@ from genno.core.computer import Computer, Key from ixmp.core.scenario import Scenario +from ixmp.report import common from . import operator -from .util import RENAME_DIMS, keys_for_quantity +from .util import keys_for_quantity class Reporter(Computer): @@ -89,7 +90,7 @@ def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": # TODO write tests for this pass - rep.add(RENAME_DIMS.get(name, name), elements) + rep.add(common.RENAME_DIMS.get(name, name), elements) return rep diff --git a/ixmp/report/util.py b/ixmp/report/util.py index caa6fa208..188956975 100644 --- a/ixmp/report/util.py +++ b/ixmp/report/util.py @@ -1,12 +1,9 @@ from functools import lru_cache, partial -from typing import Dict import pandas as pd from genno import Key -#: Dimensions to rename when extracting raw data from Scenario objects. -#: Mapping from Scenario dimension name -> preferred dimension name. -RENAME_DIMS: Dict[str, str] = {} +from ixmp.report import common def dims_for_qty(data): @@ -31,7 +28,7 @@ def dims_for_qty(data): continue # Rename dimensions - return [RENAME_DIMS.get(d, d) for d in dims] + return [common.RENAME_DIMS.get(d, d) for d in dims] def keys_for_quantity(ix_type, name, scenario): @@ -70,4 +67,11 @@ def keys_for_quantity(ix_type, name, scenario): @lru_cache(1) def get_reversed_rename_dims(): - return {v: k for k, v in RENAME_DIMS.items()} + return {v: k for k, v in common.RENAME_DIMS.items()} + + +def __getattr__(name: str): + if name == "RENAME_DIMS": + return common.RENAME_DIMS + else: + raise AttributeError(name) From 6e76165988619a4709d333fab105ea63c4543c64 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 13:06:38 +0100 Subject: [PATCH 28/37] Restore test coverage --- ixmp/backend/jdbc.py | 2 +- ixmp/core/scenario.py | 4 ++-- ixmp/report/operator.py | 2 +- ixmp/tests/core/test_scenario.py | 11 +++++++++++ ixmp/tests/report/test_reporter.py | 5 +++++ ixmp/tests/test_util.py | 4 ++++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 266d5298c..fb722f052 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -687,7 +687,7 @@ def init(self, ts, annotation): method = getattr(self.jobj, "new" + klass) try: jobj = method(ts.model, ts.scenario, *args) - except java.IxException as e: + except java.IxException as e: # pragma: no cover _raise_jexception(e) self._index_and_set_attrs(jobj, ts) diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 11dff55c8..5504a7db6 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -408,8 +408,8 @@ def items( filters = dict() elif type != ItemType.PAR: log.warning( - "Scenario.items(…, filters=…) has no effect for item type" - + repr(type.name) + "Scenario.items(…, filters=…) has no effect for item type " + + repr(type.name).lower() ) names = sorted(self._backend("list_items", str(type.name).lower())) diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index 8176a002a..571e018b0 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -258,7 +258,7 @@ def remove_ts( ts.check_out(timeseries_only=True) try: ts.remove_timeseries(data) - except Exception: + except Exception: # pragma: no cover ts.discard_changes() else: ts.commit(f"Remove time series data ({__name__}.remove_ts)") diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index 470c5c027..8c6ec1d97 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -288,6 +288,17 @@ def test_items1(self, scen, item_type, indexed_by, exp): # Function runs and yields the expected sequence of item names assert exp == list(scen.items(item_type, indexed_by=indexed_by, par_data=False)) + def test_items2(self, caplog, scen): + item_type = ixmp.ItemType.SET + + list(scen.items(item_type, filters={"foo": "bar"})) + + # Warning is logged + assert ( + "Scenario.items(…, filters=…) has no effect for item type 'set'" + in caplog.messages + ) + def test_var(self, scen): df = scen.var("x", filters={"i": ["seattle"]}) diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index afd801867..c612bd349 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -27,6 +27,11 @@ def test_configure(test_mp, test_data_path): # Configure globally; handles 'rename_dims' section configure(rename_dims={"i": "i_renamed"}) + # Test direct import + from ixmp.report import RENAME_DIMS + + assert "i" in RENAME_DIMS + # Reporting uses the RENAME_DIMS mapping of 'i' to 'i_renamed' scen = make_dantzig(test_mp) rep = Reporter.from_scenario(scen) diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 98a7fd1ed..51d94fded 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -20,6 +20,10 @@ def test_import(self): ): import ixmp.reporting.computations # type: ignore # noqa: F401 + def test_importerror(self): + with pytest.warns(DeprecationWarning), pytest.raises(ImportError): + import ixmp.reporting.foo # type: ignore # noqa: F401 + def test_check_year(): # If y is a string value, raise a Value Error. From af97a7f65ee281b680950303b4c2abcee0f642be Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 13:30:29 +0100 Subject: [PATCH 29/37] Move arg checks from JDBCBackend.init_item() to Scenario - Checking and sanitizing user input is the task of the front-end. - Consolidate Scenario.init_*() methods, parallel to has_*, *_list. --- ixmp/backend/jdbc.py | 20 +----- ixmp/core/scenario.py | 144 +++++++++++++++++------------------------- ixmp/util/__init__.py | 17 +++-- 3 files changed, 73 insertions(+), 108 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index fb722f052..65708fe62 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -908,10 +908,6 @@ def list_items(self, s, type): return to_pylist(getattr(self.jindex[s], f"get{type.title()}List")()) def init_item(self, s, type, name, idx_sets, idx_names): - # Generate index-set and index-name lists - if isinstance(idx_sets, set) or isinstance(idx_names, set): - raise TypeError("index dimension must be string or ordered lists") - # Check `idx_sets` against values hard-coded in ixmp_source try: sets = _fixed_index_sets(s.scheme)[name] @@ -927,21 +923,11 @@ def init_item(self, s, type, name, idx_sets, idx_names): f"Initialize {type} {name!r} with dimensions {idx_sets} != {sets}" ) - # Convert to Java data structure + # Convert to Java data structures idx_sets = to_jlist(idx_sets) if len(idx_sets) else None + idx_names = to_jlist(idx_names) if idx_names else idx_sets - # Handle `idx_names`, if any - if idx_names: - if len(idx_names) != len(idx_sets): - raise ValueError( - f"index names {repr(idx_names)} must have same length as index sets" - f" {repr(idx_sets)}" - ) - idx_names = to_jlist(idx_names) - else: - idx_names = idx_sets - - # Initialize the Item + # Retrieve the method that initializes the Item, something like "initializePar" func = getattr(self.jindex[s], f"initialize{type.title()}") # The constructor returns a reference to the Java Item, but these aren't exposed diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 5504a7db6..a39b4c99f 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -156,34 +156,6 @@ def _keys(self, name, key_or_keys): else: return [str(key_or_keys)] - def init_set( - self, - name: str, - idx_sets: Optional[Sequence[str]] = None, - idx_names: Optional[Sequence[str]] = None, - ) -> None: - """Initialize a new set. - - Parameters - ---------- - name : str - Name of the set. - idx_sets : sequence of str or str, optional - Names of other sets that index this set. - idx_names : sequence of str or str, optional - Names of the dimensions indexed by `idx_sets`. - - Raises - ------ - ValueError - If the set (or another object with the same *name*) already exists. - RuntimeError - If the Scenario is not checked out (see :meth:`~TimeSeries.check_out`). - """ - idx_sets = as_str_list(idx_sets) or [] - idx_names = as_str_list(idx_names) - return self._backend("init_item", "set", name, idx_sets, idx_names) - def set( self, name: str, filters: Optional[Dict[str, Sequence[str]]] = None, **kwargs ) -> Union[List[str], pd.DataFrame]: @@ -336,27 +308,6 @@ def remove_set( else: self._backend("item_delete_elements", "set", name, self._keys(name, key)) - def init_par( - self, - name: str, - idx_sets: Sequence[str], - idx_names: Optional[Sequence[str]] = None, - ) -> None: - """Initialize a new parameter. - - Parameters - ---------- - name : str - Name of the parameter. - idx_sets : sequence of str or str, optional - Names of sets that index this parameter. - idx_names : sequence of str or str, optional - Names of the dimensions indexed by `idx_sets`. - """ - idx_sets = as_str_list(idx_sets) or [] - idx_names = as_str_list(idx_names) - return self._backend("init_item", "par", name, idx_sets, idx_names) - def par( self, name: str, filters: Optional[Dict[str, Sequence[str]]] = None, **kwargs ) -> pd.DataFrame: @@ -452,6 +403,62 @@ def has_item(self, name: str, item_type=ItemType.MODEL) -> bool: #: Check whether the scenario has a variable `name`. has_var = partialmethod(has_item, item_type=ItemType.VAR) + def init_item( + self, + item_type: ItemType, + name: str, + idx_sets: Optional[Sequence[str]] = None, + idx_names: Optional[Sequence[str]] = None, + ): + """Initialize a new item `name` of type `item_type`. + + In general, user code **should** call one of :meth:`.init_set`, + :meth:`.init_par`, :meth:`.init_var`, or :meth:`.init_equ` instead of calling + this method directly. + + Parameters + ---------- + item_type : ItemType + The type of the item. + name : str + Name of the item. + idx_sets : sequence of str or str, optional + Name(s) of index sets for a 1+-dimensional item. If none are given, the item + is scalar (zero dimensional). + idx_names : sequence of str or str, optional + Names of the dimensions indexed by `idx_sets`. If given, they must be the + same length as `idx_sets`. + + Raises + ------ + ValueError + - if `idx_names` are given but do not match the length of `idx_sets`. + - if an item with the same `name`, of any `item_type`, already exists. + RuntimeError + if the Scenario is not checked out (see :meth:`~TimeSeries.check_out`). + """ + idx_sets = as_str_list(idx_sets) or [] + idx_names = as_str_list(idx_names) + + if idx_names and len(idx_names) != len(idx_sets): + raise ValueError( + f"index names {repr(idx_names)} must have same length as index sets" + f" {repr(idx_sets)}" + ) + + return self._backend( + "init_item", str(item_type.name).lower(), name, idx_sets, idx_names + ) + + #: Initialize a new equation. + init_equ = partialmethod(init_item, ItemType.EQU) + #: Initialize a new parameter. + init_par = partialmethod(init_item, ItemType.PAR) + #: Initialize a new set. + init_set = partialmethod(init_item, ItemType.SET) + #: Initialize a new variable. + init_var = partialmethod(init_item, ItemType.VAR) + def list_items( self, item_type=ItemType.MODEL, indexed_by: Optional[str] = None ) -> List[str]: @@ -581,7 +588,7 @@ def add_par( self._backend("item_set_elements", "par", name, elements) def init_scalar(self, name: str, val: Real, unit: str, comment=None) -> None: - """Initialize a new scalar. + """Initialize a new scalar and set its value. Parameters ---------- @@ -650,27 +657,6 @@ def remove_par(self, name: str, key=None) -> None: else: self._backend("item_delete_elements", "par", name, self._keys(name, key)) - def init_var( - self, - name: str, - idx_sets: Optional[Sequence[str]] = None, - idx_names: Optional[Sequence[str]] = None, - ) -> None: - """Initialize a new variable. - - Parameters - ---------- - name : str - Name of the variable. - idx_sets : sequence of str or str, optional - Name(s) of index sets for a 1+-dimensional variable. - idx_names : sequence of str or str, optional - Names of the dimensions indexed by `idx_sets`. - """ - idx_sets = as_str_list(idx_sets) or [] - idx_names = as_str_list(idx_names) - return self._backend("init_item", "var", name, idx_sets, idx_names) - def var(self, name: str, filters=None, **kwargs): """Return a dataframe of (filtered) elements for a specific variable. @@ -683,22 +669,6 @@ def var(self, name: str, filters=None, **kwargs): """ return self._backend("item_get_elements", "var", name, filters) - def init_equ(self, name: str, idx_sets=None, idx_names=None) -> None: - """Initialize a new equation. - - Parameters - ---------- - name : str - Name of the equation. - idx_sets : sequence of str or str, optional - Name(s) of index sets for a 1+-dimensional variable. - idx_names : sequence of str or str, optional - Names of the dimensions indexed by `idx_sets`. - """ - idx_sets = as_str_list(idx_sets) or [] - idx_names = as_str_list(idx_names) - return self._backend("init_item", "equ", name, idx_sets, idx_names) - def equ(self, name: str, filters=None, **kwargs) -> pd.DataFrame: """Return a dataframe of (filtered) elements for a specific equation. diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index fd8e574ef..65d8ee29e 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -7,7 +7,16 @@ from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.util import find_spec from pathlib import Path -from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, Mapping, Tuple +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, +) from urllib.parse import urlparse from warnings import warn @@ -16,7 +25,7 @@ from ixmp.backend import ItemType -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from ixmp import TimeSeries log = logging.getLogger(__name__) @@ -48,8 +57,8 @@ def logger(): return logging.getLogger("ixmp") -def as_str_list(arg, idx_names=None): - """Convert various *arg* to list of str. +def as_str_list(arg, idx_names: Optional[Iterable[str]] = None): + """Convert various `arg` to list of str. Several types of arguments are handled: From bb3084aa316ccbada9c6fce4bab8a51e87a43210 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 14:05:47 +0100 Subject: [PATCH 30/37] Add #500 to release notes --- RELEASE_NOTES.rst | 24 ++++++++++++-- doc/api.rst | 20 ++++-------- ixmp/core/scenario.py | 76 +++++++++++++++++++++++++++++++------------ 3 files changed, 85 insertions(+), 35 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 6d7a23667..6857b3859 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,11 +1,31 @@ Next release ============ -.. All changes -.. ----------- +Migration notes +--------------- +Update code that imports from the following modules: + +- :py:`ixmp.reporting` → use :py:`ixmp.report`. +- :py:`ixmp.reporting.computations` → use :py:`ixmp.report.operator`. +- :py:`ixmp.utils` → use :py:`ixmp.util`. + +Code that imports from the old locations will continue to work, but will raise :class:`DeprecationWarning`. + +All changes +----------- - Support for Python 3.7 is dropped (:pull:`492`). +- Rename :mod:`ixmp.report` and :mod:`ixmp.util` (:pull:`500`). +- New reporting operators :func:`.from_url`, :func:`.get_ts`, and :func:`.store_ts` (:pull:`500`). +- New CLI command :program:`ixmp platform copy` and :doc:`CLI documentation ` (:pull:`500`). +- New argument :py:`indexed_by=...` to :meth:`.Scenario.items` (thus :meth:`.Scenario.par_list` and similar methods) to iterate over (or list) only items that are indexed by a particular set (:issue:`402`, :pull:`500`). - New :func:`.util.discard_on_error` and matching argument to :meth:`.TimeSeries.transact` to avoid locking :class:`.TimeSeries` / :class:`.Scenario` on failed operations with :class:`.JDBCBackend` (:pull:`488`). +- Work around limitations of :class:`.JDBCBackend` (:pull:`500`): + + - Unit :py:`""` cannot be added with the Oracle driver (:issue:`425`). + - Certain items (variables) could not be initialized when providing :py:`idx_sets=...`, even if those match the sets fixed by the underlying Java code. + With this fix, a matching list is silently accepted; a different list raises :class:`NotImplementedError`. +- Improved type hinting for static typing of code that uses :mod:`ixmp` (:issue:`465`, :pull:`500`). .. _v3.7.0: diff --git a/doc/api.rst b/doc/api.rst index a3d5debfd..90f880e91 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -99,13 +99,15 @@ Scenario A Scenario is a :class:`.TimeSeries` that also contains model data, including model solution data. See the :ref:`data model documentation `. - The Scenario class provides methods to manipulate model data items: + The Scenario class provides methods to manipulate :ref:`model data items `. + In addition to generic methods (:meth:`init_item`, :meth:`items`, :meth:`list_items`, :meth:`has_item`), there are methods for each of the four item types: - Set: :meth:`init_set`, :meth:`add_set`, :meth:`set`, :meth:`remove_set`, :meth:`has_set` - Parameter: - ≥1-dimensional: :meth:`init_par`, :meth:`add_par`, :meth:`par`, :meth:`remove_par`, :meth:`par_list`, and :meth:`has_par`. - 0-dimensional: :meth:`init_scalar`, :meth:`change_scalar`, and :meth:`scalar`. + These are thin wrappers around the corresponding ``*_par`` methods, which can also be used to manipulate 0-dimensional parameters. - Variable: :meth:`init_var`, :meth:`var`, :meth:`var_list`, and :meth:`has_var`. - Equation: :meth:`init_equ`, :meth:`equ`, :meth:`equ_list`, and :meth:`has_equ`. @@ -116,33 +118,25 @@ Scenario change_scalar clone equ - equ_list - has_equ - has_par - has_set + has_item has_solution - has_var idx_names idx_sets - init_equ - init_par + init_item init_scalar - init_set - init_var + items + list_items load_scenario_data par - par_list read_excel remove_par remove_set remove_solution scalar set - set_list solve to_excel var - var_list .. _configuration: diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index a39b4c99f..cd778ceeb 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -337,7 +337,7 @@ def items( filters: Optional[Dict[str, Sequence[str]]] = None, *, indexed_by: Optional[str] = None, - par_data: bool = True, + par_data: Optional[bool] = None, ) -> Iterable[str]: """Iterate over model data items. @@ -348,13 +348,24 @@ def items( parameters. filters : dict, optional Filters for values along dimensions; same as the `filters` argument to - :meth:`par`. + :meth:`par`. Only value for :attr:`.ItemType.PAR`. + indexed_by : str, optional + If given, only iterate over items where one of the item dimensions is + `indexed_by` the set of this name. + par_data : bool, optional + If :any:`True` (the default) and `type` is :data:`.ItemType.PAR`, also + iterate over data for each parameter. Yields ------ + str + if `type` is not :attr:`.ItemType.PAR`, or `par_data` is :any:`False`: + names of items. tuple - Each tuple consists of (item name, item data). + if `type` is :attr:`.ItemType.PAR` and `par_data` is :any:`True`: + each tuple is (item name, item data). """ + # Handle `filters` argument if filters is None: filters = dict() elif type != ItemType.PAR: @@ -363,8 +374,22 @@ def items( + repr(type.name).lower() ) + # Handle `par_data` argument + if type == ItemType.PAR and par_data is None: + warn( + "using default par_data=True. In a future version of ixmp, " + "par_data=False will be default.", + FutureWarning, + 2, + ) + par_data = True + elif par_data is None: + par_data = False + + # Sorted list of items from the back end names = sorted(self._backend("list_items", str(type.name).lower())) + # Iterate over items for name in names: idx_names = set(self.idx_names(name)) idx_sets = set(self.idx_sets(name)) @@ -383,24 +408,35 @@ def items( _filters = {k: v for k, v in filters.items() if k in idx_names} yield (name, self.par(name, filters=_filters)) # type: ignore [misc] else: + # Only the name of the item yield name def has_item(self, name: str, item_type=ItemType.MODEL) -> bool: """Check whether the Scenario has an item `name` of `item_type`. + In general, user code **should** call one of :meth:`.has_equ`, :meth:`.has_par`, + :meth:`.has_set`, or :meth:`.has_var` instead of calling this method directly. + + Returns + ------- + True + if the Scenario contains an item of `item_type` with name `name`. + False + otherwise + See also -------- items """ return name in self.items(item_type, par_data=False) - #: Check whether the scenario has a equation `name`. + #: Check whether the scenario has a equation `name`. See :meth:`has_item`. has_equ = partialmethod(has_item, item_type=ItemType.EQU) - #: Check whether the scenario has a parameter `name`. + #: Check whether the scenario has a parameter `name`. See :meth:`has_item`. has_par = partialmethod(has_item, item_type=ItemType.PAR) - #: Check whether the scenario has a set `name`. + #: Check whether the scenario has a set `name`. See :meth:`has_item`. has_set = partialmethod(has_item, item_type=ItemType.SET) - #: Check whether the scenario has a variable `name`. + #: Check whether the scenario has a variable `name`. See :meth:`has_item`. has_var = partialmethod(has_item, item_type=ItemType.VAR) def init_item( @@ -450,17 +486,17 @@ def init_item( "init_item", str(item_type.name).lower(), name, idx_sets, idx_names ) - #: Initialize a new equation. + #: Initialize a new equation. See :meth:`init_item`. init_equ = partialmethod(init_item, ItemType.EQU) - #: Initialize a new parameter. + #: Initialize a new parameter. See :meth:`init_item`. init_par = partialmethod(init_item, ItemType.PAR) - #: Initialize a new set. + #: Initialize a new set. See :meth:`init_item`. init_set = partialmethod(init_item, ItemType.SET) - #: Initialize a new variable. + #: Initialize a new variable. See :meth:`init_item`. init_var = partialmethod(init_item, ItemType.VAR) def list_items( - self, item_type=ItemType.MODEL, indexed_by: Optional[str] = None + self, item_type: ItemType, indexed_by: Optional[str] = None ) -> List[str]: """List all defined items of type `item_type`. @@ -470,14 +506,14 @@ def list_items( """ return list(self.items(item_type, indexed_by=indexed_by, par_data=False)) - #: List all defined equations. - equ_list = partialmethod(list_items, item_type=ItemType.EQU) - #: List all defined parameters. - par_list = partialmethod(list_items, item_type=ItemType.PAR) - #: List all defined sets. - set_list = partialmethod(list_items, item_type=ItemType.SET) - #: List all defined variables. - var_list = partialmethod(list_items, item_type=ItemType.VAR) + #: List all defined equations. See :meth:`list_items`. + equ_list = partialmethod(list_items, ItemType.EQU) + #: List all defined parameters. See :meth:`list_items`. + par_list = partialmethod(list_items, ItemType.PAR) + #: List all defined sets. See :meth:`list_items`. + set_list = partialmethod(list_items, ItemType.SET) + #: List all defined variables. See :meth:`list_items`. + var_list = partialmethod(list_items, ItemType.VAR) def add_par( self, From 418e69aa444c73516c982a956656e813f4122ad2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 14:22:23 +0100 Subject: [PATCH 31/37] Update doc/reporting --- RELEASE_NOTES.rst | 2 +- doc/reporting.rst | 50 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 6857b3859..bfa24d411 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -16,7 +16,7 @@ All changes - Support for Python 3.7 is dropped (:pull:`492`). - Rename :mod:`ixmp.report` and :mod:`ixmp.util` (:pull:`500`). -- New reporting operators :func:`.from_url`, :func:`.get_ts`, and :func:`.store_ts` (:pull:`500`). +- New reporting operators :func:`.from_url`, :func:`.get_ts`, and :func:`.remove_ts` (:pull:`500`). - New CLI command :program:`ixmp platform copy` and :doc:`CLI documentation ` (:pull:`500`). - New argument :py:`indexed_by=...` to :meth:`.Scenario.items` (thus :meth:`.Scenario.par_list` and similar methods) to iterate over (or list) only items that are indexed by a particular set (:issue:`402`, :pull:`500`). - New :func:`.util.discard_on_error` and matching argument to :meth:`.TimeSeries.transact` to avoid locking :class:`.TimeSeries` / :class:`.Scenario` on failed operations with :class:`.JDBCBackend` (:pull:`488`). diff --git a/doc/reporting.rst b/doc/reporting.rst index afaf8385a..bdacff31f 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -61,17 +61,12 @@ The following top-level objects from :mod:`genno` may also be imported from .. autosummary:: ~Computer.add - ~Computer.add_file - ~Computer.add_product ~Computer.add_queue ~Computer.add_single - ~Computer.aggregate ~Computer.apply ~Computer.check_keys ~Computer.configure - ~Computer.convert_pyam ~Computer.describe - ~Computer.disaggregate ~Computer.full_key ~Computer.get ~Computer.infer_keys @@ -79,6 +74,17 @@ The following top-level objects from :mod:`genno` may also be imported from ~Computer.visualize ~Computer.write + The following methods are deprecated; equivalent or better functionality is available through :meth:`.Computer.add`. + See the genno documentation for each method for suggested changes/migrations. + + .. autosummary:: + + ~Computer.add_file + ~Computer.add_product + ~Computer.aggregate + ~Computer.convert_pyam + ~Computer.disaggregate + .. _reporting-config: Configuration @@ -138,9 +144,12 @@ Operators .. autosummary:: data_for_quantity + from_url + get_ts map_as_qty update_scenario store_ts + remove_ts Basic operators are defined by :mod:`genno.operator` and its compatibility modules; see there for details: @@ -172,5 +181,36 @@ Operators Utilities ========= +.. currentmodule:: ixmp.report.common + +.. autodata:: RENAME_DIMS + + User code **should** avoid directly manipulating :data:`RENAME_DIMS`. + Instead, call :func:`.configure`: + + .. code-block:: python + + # Rename dimension "long_dimension_name" to "ldn" + configure(rename_dims={"long_dimension_name": "ldn"}) + + As well, importing the variable into the global namespace of another module creates a copy of the dictionary that may become out of sync with other changes. + Thus, instead of: + + .. code-block:: python + + from ixmp.report import RENAME_DIMS + + def my_operator(...): + # Code that references RENAME_DIMS + + Do this: + + .. code-block:: python + + def my_operator(...): + from ixmp.report import common + + # Code that references common.RENAME_DIMS + .. automodule:: ixmp.report.util :members: From f1f98571116cda770344d8a45b03b599d7a96fa2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 17 Nov 2023 17:03:27 +0100 Subject: [PATCH 32/37] Adjust .util import in tests --- ixmp/tests/test_util.py | 44 ++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 51d94fded..4c9c8f1de 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -7,7 +7,7 @@ import pytest from pytest import mark, param -from ixmp import Scenario, utils +from ixmp import Scenario, util from ixmp.testing import make_dantzig, populate_test_platform @@ -31,21 +31,21 @@ def test_check_year(): y1 = "a" s1 = "a" with pytest.raises(ValueError): - assert utils.check_year(y1, s1) + assert util.check_year(y1, s1) # If y = None. y2 = None s2 = None - assert utils.check_year(y2, s2) is None + assert util.check_year(y2, s2) is None # If y is integer. y3 = 4 s3 = 4 - assert utils.check_year(y3, s3) is True + assert util.check_year(y3, s3) is True def test_diff_identical(test_mp): @@ -54,12 +54,12 @@ def test_diff_identical(test_mp): scen_b = make_dantzig(test_mp) # Compare identical scenarios: produces data of same length - for name, df in utils.diff(scen_a, scen_b): - data_a = utils.maybe_convert_scalar(scen_a.par(name)) + for name, df in util.diff(scen_a, scen_b): + data_a = util.maybe_convert_scalar(scen_a.par(name)) assert len(data_a) == len(df) # Compare identical scenarios, with filters - iterator = utils.diff(scen_a, scen_b, filters=dict(i=["seattle"])) + iterator = util.diff(scen_a, scen_b, filters=dict(i=["seattle"])) for (name, df), (exp_name, N) in zip(iterator, [("a", 1), ("d", 3)]): assert exp_name == name and len(df) == N @@ -109,14 +109,14 @@ def test_diff_data(test_mp): exp_d = exp_d.astype(dict(_merge=merge_cat)) # Compare different scenarios without filters - for name, df in utils.diff(scen_a, scen_b): + for name, df in util.diff(scen_a, scen_b): if name == "b": pdt.assert_frame_equal(exp_b, df) elif name == "d": pdt.assert_frame_equal(exp_d, df) # Compare different scenarios with filters - iterator = utils.diff(scen_a, scen_b, filters=dict(j=["chicago"])) + iterator = util.diff(scen_a, scen_b, filters=dict(j=["chicago"])) for name, df in iterator: # Same as above, except only the filtered rows should appear if name == "b": @@ -139,11 +139,11 @@ def test_diff_items(test_mp): scen_b.remove_par("d") # Compare different scenarios without filters - for name, df in utils.diff(scen_a, scen_b): + for name, df in util.diff(scen_a, scen_b): pass # No check on the contents # Compare different scenarios with filters - iterator = utils.diff(scen_a, scen_b, filters=dict(j=["chicago"])) + iterator = util.diff(scen_a, scen_b, filters=dict(j=["chicago"])) for name, df in iterator: pass # No check of the contents @@ -163,7 +163,7 @@ def test_discard_on_error(caplog, test_mp): # Catch the deliberately-raised exception so that the test passes with pytest.raises(KeyError): # Trigger KeyError and the discard_on_error() behaviour - with utils.discard_on_error(s): + with util.discard_on_error(s): s.add_par("d", pd.DataFrame([["foo", "bar", 1.0, "kg"]])) # Exception was caught and logged @@ -176,13 +176,13 @@ def test_discard_on_error(caplog, test_mp): # Re-load the mp and the scenario with pytest.raises(RuntimeError): # Fails because the connection to test_mp was closed by discard_on_error() - s2 = Scenario(test_mp, **utils.parse_url(url)[1]) + s2 = Scenario(test_mp, **util.parse_url(url)[1]) # Reopen the connection test_mp.open_db() # Now the scenario can be reloaded - s2 = Scenario(test_mp, **utils.parse_url(url)[1]) + s2 = Scenario(test_mp, **util.parse_url(url)[1]) assert s2 is not s # Different object instance than above # Data modification above was discarded by discard_on_error() @@ -191,17 +191,17 @@ def test_discard_on_error(caplog, test_mp): def test_filtered(): df = pd.DataFrame() - assert df is utils.filtered(df, filters=None) + assert df is util.filtered(df, filters=None) def test_isscalar(): with pytest.warns(DeprecationWarning): - assert False is utils.isscalar([3, 4]) + assert False is util.isscalar([3, 4]) def test_logger_deprecated(): with pytest.warns(DeprecationWarning): - utils.logger() + util.logger() m_s = dict(model="m", scenario="s") @@ -245,7 +245,7 @@ def test_logger_deprecated(): @pytest.mark.parametrize("url, p, s", URLS) def test_parse_url(url, p, s): - platform_info, scenario_info = utils.parse_url(url) + platform_info, scenario_info = util.parse_url(url) # Expected platform and scenario information is returned assert platform_info == p @@ -271,7 +271,7 @@ def test_format_scenario_list(test_mp_f): "2 scenario name(s)", "2 (model, scenario) combination(s)", "4 total scenarios", - ] == utils.format_scenario_list(mp) + ] == util.format_scenario_list(mp) # With as_url=True assert list( @@ -282,17 +282,17 @@ def test_format_scenario_list(test_mp_f): "ixmp://{}/canning problem/standard#2", ], ) - ) == utils.format_scenario_list(mp, as_url=True) + ) == util.format_scenario_list(mp, as_url=True) def test_maybe_commit(caplog, test_mp): s = Scenario(test_mp, "maybe_commit", "maybe_commit", version="new") # A new Scenario is not committed, so this works - assert utils.maybe_commit(s, True, message="foo") is True + assert util.maybe_commit(s, True, message="foo") is True # *s* is already commited. No commit is performed, but the function call # succeeds and a message is logged caplog.set_level(logging.INFO, logger="ixmp") - assert utils.maybe_commit(s, True, message="foo") is False + assert util.maybe_commit(s, True, message="foo") is False assert caplog.messages[-1].startswith("maybe_commit() didn't commit: ") diff --git a/pyproject.toml b/pyproject.toml index 520346414..8b573c047 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,7 @@ exclude_also = [ # Imports only used by type checkers "if TYPE_CHECKING:", ] -omit = ["ixmp/utils/sphinx_linkcode_github.py"] +omit = ["ixmp/util/sphinx_linkcode_github.py"] [tool.isort] profile = "black" From 12efc02285261a0b138249f4e9fb77f7a1fd66cc Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 20 Nov 2023 23:13:17 +0100 Subject: [PATCH 33/37] Handle LP status 5 exception in GAMSModel --- ixmp/backend/jdbc.py | 5 +++- ixmp/model/gams.py | 66 ++++++++++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 65708fe62..7ba6b8b5d 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -556,7 +556,10 @@ def read_file(self, path, item_type: ItemType, **kwargs): if len(kwargs): raise ValueError(f"extra keyword arguments {kwargs}") - self.jindex[ts].readSolutionFromGDX(*args) + try: + self.jindex[ts].readSolutionFromGDX(*args) + except java.Exception as e: + _raise_jexception(e) self.cache_invalidate(ts) else: diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 131cb25df..5e834aa26 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -5,7 +5,7 @@ import tempfile from copy import copy from pathlib import Path -from subprocess import CalledProcessError, check_call +from subprocess import CalledProcessError, run from typing import Any, MutableMapping from ixmp.backend import ItemType @@ -188,21 +188,30 @@ def __init__(self, name_=None, **model_options): # Not set; use `quiet` to determine the value self.gams_args.append(f"LogOption={'2' if self.quiet else '4'}") - def format_exception(self, exc, model_file): + def format_exception(self, exc, model_file, backend_class): """Format a user-friendly exception when GAMS errors.""" - msg = [ - f"GAMS errored with return code {exc.returncode}:", - # Convert a Windows return code >256 to its equivalent on *nix platforms - f" {RETURN_CODE[exc.returncode % 256]}", - "", - "For details, see the terminal output above, plus:", - f"Input data: {self.in_file}", - ] + lst_file = Path(self.cwd).joinpath(model_file.name).with_suffix(".lst") + lp_5 = "LP status (5): optimal with unscaled infeasibilities" + + if getattr(exc, "returncode", 0) > 0: + # Convert a Windows return code >256 to its POSIX equivalent + msg = [ + f"GAMS errored with return code {exc.returncode}:", + f" {RETURN_CODE[exc.returncode % 256]}", + ] + elif lst_file.exists() and lp_5 in lst_file.read_text(): # pragma: no cover + msg = [ + "GAMS returned 0 but indicated:", + f" {lp_5}", + f"and {backend_class.__name__} could not read the solution.", + ] # Add a reference to the listing file, if it exists - lst_file = Path(self.cwd).joinpath(model_file.name).with_suffix(".lst") - if lst_file.exists(): - msg.insert(-1, f"Listing : {lst_file}") + msg.extend( + ["", "For details, see the terminal output above, plus:"] + + ([f"Listing : {lst_file}"] if lst_file.exists() else []) + + [f"Input data: {self.in_file}"] + ) return ModelError("\n".join(msg)) @@ -290,21 +299,24 @@ def run(self, scenario): try: # Invoke GAMS - check_call(command, shell=os.name == "nt", cwd=self.cwd) - except CalledProcessError as exc: + run(command, shell=os.name == "nt", cwd=self.cwd, check=True) + + # Read model solution + scenario.platform._backend.read_file( + self.out_file, + ItemType.MODEL, + **s_arg, + check_solution=self.check_solution, + comment=self.comment or "", + equ_list=as_str_list(self.equ_list) or [], + var_list=as_str_list(self.var_list) or [], + ) + except (CalledProcessError, RuntimeError) as exc: + # CalledProcessError from run(); RuntimeError from read_file() # Do not remove self.temp_dir; the user may want to inspect the GDX file - raise self.format_exception(exc, model_file) from None - - # Read model solution - scenario.platform._backend.read_file( - self.out_file, - ItemType.MODEL, - **s_arg, - check_solution=self.check_solution, - comment=self.comment or "", - equ_list=as_str_list(self.equ_list) or [], - var_list=as_str_list(self.var_list) or [], - ) + raise self.format_exception( + exc, model_file, scenario.platform._backend.__class__ + ) from None # Finished: remove the temporary directory, if any self.remove_temp_dir() From 13369b33b53baaec13ad5279c6c1b7edcf2c71d2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 21 Nov 2023 11:09:25 +0100 Subject: [PATCH 34/37] Add #98 to release notes --- RELEASE_NOTES.rst | 2 ++ doc/api-model.rst | 3 +++ 2 files changed, 5 insertions(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index bfa24d411..aae2f0f90 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -25,6 +25,8 @@ All changes - Unit :py:`""` cannot be added with the Oracle driver (:issue:`425`). - Certain items (variables) could not be initialized when providing :py:`idx_sets=...`, even if those match the sets fixed by the underlying Java code. With this fix, a matching list is silently accepted; a different list raises :class:`NotImplementedError`. + - When a :class:`.GAMSModel` is solved with an LP status of 5 (optimal, but with infeasibilities after unscaling), :class:`.JDBCBackend` would attempt to read the output GDX file and fail, leading to an uninformative error message (:issue:`98`). + Now :class:`.ModelError` is raised describing the situation. - Improved type hinting for static typing of code that uses :mod:`ixmp` (:issue:`465`, :pull:`500`). .. _v3.7.0: diff --git a/doc/api-model.rst b/doc/api-model.rst index 94a2cfa26..7c9ad78ff 100644 --- a/doc/api-model.rst +++ b/doc/api-model.rst @@ -31,6 +31,9 @@ Model API .. currentmodule:: ixmp.model.base +.. automodule:: ixmp.model.base + :members: ModelError + .. autoclass:: ixmp.model.base.Model :members: name, __init__, run, initialize, initialize_items, enforce From 35602139baeecb74a2e7705c015165c6aff0fee1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 21 Nov 2023 11:09:57 +0100 Subject: [PATCH 35/37] Set Sphinx option nitpicky=True --- doc/conf.py | 8 ++------ ixmp/core/scenario.py | 4 ++-- ixmp/model/base.py | 2 +- ixmp/report/util.py | 2 +- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 89e2496ca..0c34efdfb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -39,12 +39,8 @@ # html_extra_path. exclude_patterns = ["_build", "README.rst", "Thumbs.db", ".DS_Store"] -nitpick_ignore_regex = { - # These occur because there is no .. py:module:: directive for the *top-level* - # module or package in the respective documentation and inventories. - # TODO Remove once the respective docs are fixed - ("py:mod", "message_ix"), -} +# Warn about *all* references where the target cannot be found +nitpicky = True # A string of reStructuredText that will be included at the beginning of every source # file that is read. diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index cd778ceeb..dc539dea8 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -353,7 +353,7 @@ def items( If given, only iterate over items where one of the item dimensions is `indexed_by` the set of this name. par_data : bool, optional - If :any:`True` (the default) and `type` is :data:`.ItemType.PAR`, also + If :any:`True` (the default) and `type` is :attr:`.ItemType.PAR`, also iterate over data for each parameter. Yields @@ -454,7 +454,7 @@ def init_item( Parameters ---------- - item_type : ItemType + item_type : .ItemType The type of the item. name : str Name of the item. diff --git a/ixmp/model/base.py b/ixmp/model/base.py index 2f7587693..5332fbfd2 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -13,7 +13,7 @@ class ModelError(Exception): - """Error in model code, e.g. :meth:`.Model.run`.""" + """Error in model code—that is, :meth:`.Model.run` or other code called by it.""" class Model(ABC): diff --git a/ixmp/report/util.py b/ixmp/report/util.py index 188956975..3e718cc80 100644 --- a/ixmp/report/util.py +++ b/ixmp/report/util.py @@ -12,7 +12,7 @@ def dims_for_qty(data): If *data* is a :class:`pandas.DataFrame`, its columns are processed; otherwise it must be a list. - :data:`RENAME_DIMS` is used to rename dimensions. + :data:`.RENAME_DIMS` is used to rename dimensions. """ if isinstance(data, pd.DataFrame): # List of the dimensions From 3f99fc4c64c18b01b74d6d8527f06e306ebc2580 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 21 Nov 2023 11:34:46 +0100 Subject: [PATCH 36/37] Improve test coverage - Add a context-manager form of .jdbc._raise_jexception. --- ixmp/__init__.py | 5 ++-- ixmp/backend/jdbc.py | 45 +++++++++++++++----------------- ixmp/core/scenario.py | 4 +-- ixmp/testing/resource.py | 2 +- ixmp/tests/backend/test_jdbc.py | 5 ++++ ixmp/tests/core/test_scenario.py | 13 ++++++--- ixmp/tests/report/test_util.py | 5 ++++ ixmp/tests/test_util.py | 7 +++++ ixmp/util/__init__.py | 4 +-- 9 files changed, 55 insertions(+), 35 deletions(-) create mode 100644 ixmp/tests/report/test_util.py diff --git a/ixmp/__init__.py b/ixmp/__init__.py index acfc57117..a3c881884 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -69,8 +69,9 @@ def __getattr__(name): if name == "utils": - import ixmp.util + # Import via the old name to trigger DeprecatedPathFinder + import ixmp.utils as util # type: ignore [import-not-found] - return ixmp.util + return util else: raise AttributeError(name) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 7ba6b8b5d..637783bd1 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -4,6 +4,7 @@ import re from collections import ChainMap from collections.abc import Iterable, Sequence +from contextlib import contextmanager from copy import copy from functools import lru_cache from pathlib import Path, PurePosixPath @@ -141,6 +142,15 @@ def _raise_jexception(exc, msg="unhandled Java exception: "): raise RuntimeError(msg) from None +@contextmanager +def _handle_jexception(): + """Context manager form of :func:`_raise_jexception`.""" + try: + yield + except java.Exception as e: + _raise_jexception(e) + + @lru_cache def _fixed_index_sets(scheme: str) -> Mapping[str, List[str]]: """Return index sets for items that are fixed in the Java code. @@ -475,10 +485,8 @@ def get_scenario_names(self) -> Generator[str, None, None]: def get_scenarios(self, default, model, scenario): # List> - try: + with _handle_jexception(): scenarios = self.jobj.getScenarioList(default, model, scenario) - except java.IxException as e: - _raise_jexception(e) for s in scenarios: data = [] @@ -537,7 +545,7 @@ def read_file(self, path, item_type: ItemType, **kwargs): if path.suffix == ".gdx" and item_type is ItemType.MODEL: kw = {"check_solution", "comment", "equ_list", "var_list"} - if not isinstance(ts, Scenario): + if not isinstance(ts, Scenario): # pragma: no cover raise ValueError("read from GDX requires a Scenario object") elif set(kwargs.keys()) != kw: raise ValueError( @@ -556,10 +564,8 @@ def read_file(self, path, item_type: ItemType, **kwargs): if len(kwargs): raise ValueError(f"extra keyword arguments {kwargs}") - try: + with _handle_jexception(): self.jindex[ts].readSolutionFromGDX(*args) - except java.Exception as e: - _raise_jexception(e) self.cache_invalidate(ts) else: @@ -601,7 +607,7 @@ def write_file(self, path, item_type: ItemType, **kwargs): if path.suffix == ".gdx" and item_type is ItemType.SET | ItemType.PAR: if len(filters): raise NotImplementedError("write to GDX with filters") - elif not isinstance(ts, Scenario): + elif not isinstance(ts, Scenario): # pragma: no cover raise ValueError("write to GDX requires a Scenario object") # include_var_equ=False -> do not include variables/equations in GDX @@ -688,10 +694,8 @@ def init(self, ts, annotation): # Call either newTimeSeries or newScenario method = getattr(self.jobj, "new" + klass) - try: + with _handle_jexception(): jobj = method(ts.model, ts.scenario, *args) - except java.IxException as e: # pragma: no cover - _raise_jexception(e) self._index_and_set_attrs(jobj, ts) @@ -703,12 +707,11 @@ def get(self, ts): # either getTimeSeries or getScenario method = getattr(self.jobj, "get" + ts.__class__.__name__) - try: + + # Re-raise as a ValueError for bad model or scenario name, or other + with _handle_jexception(): # Either the 2- or 3- argument form, depending on args jobj = method(*args) - except java.IxException as e: - # Re-raise as a ValueError for bad model or scenario name, or other - _raise_jexception(e) self._index_and_set_attrs(jobj, ts) @@ -719,10 +722,8 @@ def del_ts(self, ts): self.gc() def check_out(self, ts, timeseries_only): - try: + with _handle_jexception(): self.jindex[ts].checkOut(timeseries_only) - except java.IxException as e: - _raise_jexception(e) def commit(self, ts, comment): try: @@ -1120,10 +1121,8 @@ def get_meta( if version is not None: version = java.Long(version) - try: + with _handle_jexception(): meta = self.jobj.getMeta(model, scenario, version, strict) - except java.IxException as e: - _raise_jexception(e) return {entry.getKey(): _unwrap(entry.getValue()) for entry in meta.entrySet()} @@ -1142,10 +1141,8 @@ def set_meta( for k, v in meta.items(): jmeta.put(str(k), _wrap(v)) - try: + with _handle_jexception(): self.jobj.setMeta(model, scenario, version, jmeta) - except java.IxException as e: - _raise_jexception(e) def remove_meta( self, diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index dc539dea8..1572974ce 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -378,7 +378,7 @@ def items( if type == ItemType.PAR and par_data is None: warn( "using default par_data=True. In a future version of ixmp, " - "par_data=False will be default.", + "par_data=False will be the default.", FutureWarning, 2, ) @@ -478,7 +478,7 @@ def init_item( if idx_names and len(idx_names) != len(idx_sets): raise ValueError( - f"index names {repr(idx_names)} must have same length as index sets" + f"index names {repr(idx_names)} must have the same length as index sets" f" {repr(idx_sets)}" ) diff --git a/ixmp/testing/resource.py b/ixmp/testing/resource.py index c7e9da121..ee9a5f6fb 100644 --- a/ixmp/testing/resource.py +++ b/ixmp/testing/resource.py @@ -7,7 +7,7 @@ import resource has_resource_module = True -except ImportError: +except ImportError: # pragma: no cover # Windows has_resource_module = False diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index afd254e94..b17147ef6 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -201,6 +201,11 @@ def test_write_file(self, tmp_path, be): with pytest.raises(NotImplementedError): be.write_file(tmp_path / "test.csv", ixmp.ItemType.ALL, filters={}) + # Specific to JDBCBackend + def test_gc(self, monkeypatch, be): + monkeypatch.setattr(ixmp.backend.jdbc, "_GC_AGGRESSIVE", True) + be.gc() + def test_exceptions(test_mp): """Ensure that Python exceptions are raised for some actions.""" diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index 8c6ec1d97..886ed73f3 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -147,7 +147,7 @@ def test_init_set(self, scen): with pytest.raises(ValueError, match="'foo' already exists"): scen.init_set("foo") - def test_init_par(self, scen): + def test_init_par(self, scen) -> None: scen = scen.clone(keep_solution=False) scen.check_out() @@ -157,6 +157,10 @@ def test_init_par(self, scen): # Return type of idx_sets is still list assert scen.idx_sets("foo") == ["i", "j"] + # Mismatched sets and names + with pytest.raises(ValueError, match="must have the same length"): + scen.init_par("bar", idx_sets=("i", "j"), idx_names=("a", "b", "c")) + def test_init_scalar(self, scen): scen2 = scen.clone(keep_solution=False) scen2.check_out() @@ -246,11 +250,12 @@ def test_items0(self, scen): iterator = scen.items() # next() can be called → an iterator was returned - next(iterator) + with pytest.warns(FutureWarning, match="par_data=False will be the default"): + next(iterator) # Iterator returns the expected parameter names exp = ["a", "b", "d", "f"] - for i, (name, data) in enumerate(scen.items()): + for i, (name, data) in enumerate(scen.items(par_data=True)): # Name is correct in the expected order assert exp[i] == name # Data is one of the return types of .par() @@ -260,7 +265,7 @@ def test_items0(self, scen): assert i == 3 # With filters - iterator = scen.items(filters=dict(i=["seattle"])) + iterator = scen.items(filters=dict(i=["seattle"]), par_data=True) exp = [("a", 1), ("d", 3)] for i, (name, data) in enumerate(iterator): # Name is correct in the expected order diff --git a/ixmp/tests/report/test_util.py b/ixmp/tests/report/test_util.py new file mode 100644 index 000000000..47ff277ae --- /dev/null +++ b/ixmp/tests/report/test_util.py @@ -0,0 +1,5 @@ +def test_import_rename_dims(): + """RENAME_DIMS can be imported from .report.util, though defined in .common.""" + from ixmp.report.util import RENAME_DIMS + + assert isinstance(RENAME_DIMS, dict) diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 4c9c8f1de..12c71a694 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -20,6 +20,13 @@ def test_import(self): ): import ixmp.reporting.computations # type: ignore # noqa: F401 + @pytest.mark.filterwarnings("ignore") + def test_import1(self): + """utils can be imported from ixmp, but raises DeprecationWarning.""" + from ixmp import utils + + assert "diff" in dir(utils) + def test_importerror(self): with pytest.warns(DeprecationWarning), pytest.raises(ImportError): import ixmp.reporting.foo # type: ignore # noqa: F401 diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index 65d8ee29e..f6c1798a9 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -115,8 +115,8 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: """ # Iterators; index 0 corresponds to `a`, 1 to `b` items = [ - a.items(filters=filters, type=ItemType.PAR), - b.items(filters=filters, type=ItemType.PAR), + a.items(filters=filters, type=ItemType.PAR, par_data=True), + b.items(filters=filters, type=ItemType.PAR, par_data=True), ] # State variables for loop name = ["", ""] From a8819d1b8cfb987404a819acd2df1e3066718851 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 21 Nov 2023 12:37:53 +0100 Subject: [PATCH 37/37] Remove Sphinx build step from "pytest" CI workflow --- .github/workflows/pytest.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 7b8a95dbe..e6217598c 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -127,14 +127,6 @@ jobs: - name: Run test suite using pytest run: pytest ixmp -m "not performance" --verbose -rA --cov-report=xml --color=yes - - name: Test documentation build using Sphinx - env: - # For pull_request triggers, GitHub creates a temporary merge commit - # with a hash that does not match the head of the branch. Tell it which - # branch to use. - SPHINXOPTS: -D linkcode_github_remote_head=${{ github.head_ref }} - run: make --directory=doc html - - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v3 @@ -146,6 +138,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + with: + python-version: "3.x" - name: Force recreation of pre-commit virtual environment for mypy if: github.event_name == 'schedule'