diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..6d734d2c5 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + # Only update major versions + ignore: + - dependency-name: "*" + update-types: + - "version-update:semver-minor" + - "version-update:semver-patch" + # Below config mirrors the example at + # https://github.com/dependabot/dependabot-core/blob/main/.github/dependabot.yml + directory: "/" + schedule: + interval: "weekly" + day: "sunday" + time: "16:00" + groups: + all-actions: + patterns: [ "*" ] + assignees: + - "glatterf42" + - "khaeru" + labels: + - "dependencies" + reviewers: + - "glatterf42" + - "khaeru" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ab5832c68..bff43284e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,27 +1,32 @@ **Required:** write a single sentence that describes the changes made by this PR. - + ## How to review **Required:** describe specific things that reviewer(s) must do, in order to ensure that the PR achieves its goal. -If no review is required, write “No review:” and describe why. +If no review is required, write “No review: …” and describe why. - [ ] Continuous integration checks all ✅ - + - [ ] Add or expand tests; coverage checks both ✅ - [ ] Add, expand, or update documentation. - [ ] Update release notes. @@ -47,7 +53,7 @@ For example, one or more of: To do this, add a single line at the TOP of the “Next release” section of RELEASE_NOTES.rst, where '999' is the GitHub pull request number: - - :pull:`999`: Title or single-sentence description from above. + - Title or single-sentence description from above (:pull:`999`:). Commit with a message like “Add #999 to release notes” --> diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index e7b71eb40..64d0cff41 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -141,12 +141,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - with: - python-version: "3.x" + with: { python-version: "3.12" } - name: Force recreation of pre-commit virtual environment for mypy if: github.event_name == 'schedule' run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true env: { GH_TOKEN: "${{ github.token }}" } - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b15535660..05ffac9f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,16 +10,17 @@ repos: language: python entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@" additional_dependencies: - - mypy >= 1.8.0 + - mypy >= 1.9.0 - genno - GitPython - nbclient + - pandas-stubs - pytest - sphinx - xarray args: ["."] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.14 + rev: v0.3.4 hooks: - id: ruff - id: ruff-format diff --git a/doc/conf.py b/doc/conf.py index f04ae1efb..a705e4c2c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -19,7 +19,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions coming # with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "ixmp.util.sphinx_linkcode_github", + # First party "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.coverage", @@ -29,6 +29,9 @@ "sphinx.ext.mathjax", "sphinx.ext.napoleon", "sphinx.ext.todo", + # Others + "ixmp.util.sphinx_linkcode_github", + "genno.compat.sphinx.rewrite_refs", "sphinxcontrib.bibtex", ] @@ -77,6 +80,14 @@ # The theme to use for HTML and HTML Help pages. html_theme = "sphinx_rtd_theme" +# -- Options for genno.compat.sphinx.rewrite_refs -------------------------------------- + +reference_aliases = { + r"(genno\.|)Quantity": "genno.core.attrseries.AttrSeries", + "AnyQuantity": ":data:`genno.core.quantity.AnyQuantity`", +} + + # -- Options for sphinx.ext.extlinks --------------------------------------------------- extlinks = { @@ -121,7 +132,7 @@ def local_inv(name: str, *parts: str) -> Optional[str]: "python": ("https://docs.python.org/3/", None), "sparse": ("https://sparse.pydata.org/en/stable/", None), "sphinx": ("https://www.sphinx-doc.org/en/master/", None), - "xarray": ("https://xarray.pydata.org/en/stable/", None), + "xarray": ("https://docs.xarray.dev/en/stable", None), } # -- Options for sphinx.ext.linkcode / ixmp.util.sphinx_linkcode_github ---------------- diff --git a/doc/reporting.rst b/doc/reporting.rst index a22a0c189..5db6466c7 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -143,7 +143,10 @@ Operators .. automodule:: ixmp.report.operator :members: - :mod:`ixmp.report` defines these operators: + More than 30 operators are defined by :mod:`genno.operator` and its compatibility modules including :mod:`genno.compat.plotnine` and :mod:`genno.compat.sdmx`. + See the genno documentation for details. + + :mod:`ixmp.report` defines these additional operators: .. autosummary:: @@ -151,36 +154,9 @@ Operators from_url get_ts map_as_qty - update_scenario - store_ts remove_ts - - Basic operators are defined by :mod:`genno.operator` and its compatibility modules; see there for details: - - .. autosummary:: - - ~genno.compat.plotnine.Plot - ~genno.operator.add - ~genno.operator.aggregate - ~genno.operator.apply_units - ~genno.compat.pyam.operator.as_pyam - ~genno.operator.broadcast_map - ~genno.operator.combine - ~genno.operator.concat - ~genno.operator.disaggregate_shares - ~genno.operator.div - ~genno.operator.group_sum - ~genno.operator.interpolate - ~genno.operator.load_file - ~genno.operator.mul - ~genno.operator.pow - ~genno.operator.product - ~genno.operator.relabel - ~genno.operator.rename_dims - ~genno.operator.ratio - ~genno.operator.select - ~genno.operator.sum - ~genno.operator.write_report + store_ts + update_scenario Utilities ========= diff --git a/ixmp/backend/__init__.py b/ixmp/backend/__init__.py index cd2796e20..4c595f3f7 100644 --- a/ixmp/backend/__init__.py +++ b/ixmp/backend/__init__.py @@ -1,4 +1,5 @@ """Backend API.""" + from enum import IntFlag from typing import Dict, List, Type, Union diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py index d862cbc42..92b6e2c1a 100644 --- a/ixmp/backend/io.py +++ b/ixmp/backend/io.py @@ -112,7 +112,9 @@ def s_write_excel(be, s, path, item_type, filters=None, max_row=None): sheet_name = name + (f"({sheet_num})" if sheet_num > 1 else "") # Subset the data (only on rows, if a DataFrame) and write - data.iloc[first_row:last_row].to_excel(writer, sheet_name, index=False) + data.iloc[first_row:last_row].to_excel( + writer, sheet_name=sheet_name, index=False + ) # Discard entries that were not written for name in omitted: @@ -172,7 +174,7 @@ def maybe_init_item(scenario, ix_type, name, new_idx, path): raise ValueError from None -# FIXME reduce complexity from 26 to <=15 +# FIXME reduce complexity 22 → ≤13 def s_read_excel( # noqa: C901 be, s, path, add_units=False, init_items=False, commit_steps=False ): diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index f93b4069b..671276168 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -965,7 +965,7 @@ def item_index(self, s, name, sets_or_names): jitem = self._get_item(s, "item", name, load=False) return list(getattr(jitem, f"getIdx{sets_or_names.title()}")()) - # FIXME reduce complexity from 19 to <=15 + # FIXME reduce complexity 18 → ≤13 def item_get_elements(self, s, type, name, filters=None): # noqa: C901 if filters: # Convert filter elements to strings diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index 8218729dd..10b770b27 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -9,7 +9,7 @@ from ixmp.backend import BACKENDS, FIELDS, ItemType from ixmp.util import as_str_list -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from ixmp.backend.base import Backend @@ -235,11 +235,11 @@ def export_timeseries_data( "model or scenario." ) filters = { - "model": as_str_list(model) or [], - "scenario": as_str_list(scenario) or [], - "variable": as_str_list(variable) or [], - "unit": as_str_list(unit) or [], - "region": as_str_list(region) or [], + "model": as_str_list(model), + "scenario": as_str_list(scenario), + "variable": as_str_list(variable), + "unit": as_str_list(unit), + "region": as_str_list(region), "default": default, "export_all_runs": export_all_runs, } diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 5201e634f..26de09b20 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -1,10 +1,20 @@ import logging from functools import partialmethod -from itertools import repeat, zip_longest +from itertools import zip_longest from numbers import Real from os import PathLike from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Union +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + MutableSequence, + Optional, + Sequence, + Union, +) from warnings import warn import pandas as pd @@ -176,12 +186,12 @@ def set( """ return self._backend("item_get_elements", "set", name, filters) - # FIXME reduce complexity from 17 to <=15 + # FIXME reduce complexity 18 → ≤13 def add_set( # noqa: C901 self, name: str, key: Union[str, Sequence[str], Dict, pd.DataFrame], - comment: Optional[str] = None, + comment: Union[str, Sequence[str], None] = None, ) -> None: """Add elements to an existing set. @@ -209,83 +219,93 @@ def add_set( # noqa: C901 if isinstance(key, list) and len(key) == 0: return # No elements to add + elif comment and isinstance(key, (dict, pd.DataFrame)) and "comment" in key: + # Ambiguous arguments + raise ValueError("ambiguous; both key['comment'] and comment= given") # Get index names for set *name*, may raise KeyError idx_names = self.idx_names(name) + # List of keys + keys: MutableSequence[Union[str, MutableSequence[str]]] = [] + # List of comments for each key + comments: List[Optional[str]] = [] + # Check arguments and convert to two lists: keys and comments if len(idx_names) == 0: # Basic set. Keys must be strings. if isinstance(key, (dict, pd.DataFrame)): raise TypeError( - f"keys for basic set {repr(name)} must be str or list of str; got " + f"keys for basic set {name!r} must be str or list of str; got " f"{type(key)}" ) # Ensure keys is a list of str - keys = as_str_list(key) + keys.extend(as_str_list(key)) else: # Set defined over 1+ other sets - - # Check for ambiguous arguments - if comment and isinstance(key, (dict, pd.DataFrame)) and "comment" in key: - raise ValueError("ambiguous; both key['comment'] and comment= given") - if isinstance(key, pd.DataFrame): # DataFrame of key values and perhaps comments try: # Pop a 'comment' column off the DataFrame, convert to list - comment = key.pop("comment").to_list() + comments.extend(key.pop("comment")) except KeyError: pass # Convert key to list of list of key values - keys = [] for row in key.to_dict(orient="records"): keys.append(as_str_list(row, idx_names=idx_names)) elif isinstance(key, dict): # Dict of lists of key values # Pop a 'comment' list from the dict - comment = key.pop("comment", None) + comments.extend(key.pop("comment", [])) # Convert to list of list of key values - keys = list(map(as_str_list, zip(*[key[i] for i in idx_names]))) + keys.extend(map(as_str_list, zip(*[key[i] for i in idx_names]))) + elif isinstance(key, str) and len(idx_names) == 1: + # Bare key given for a 1D set; wrap for convenience + keys.append([key]) elif isinstance(key[0], str): # List of key values; wrap - keys = [as_str_list(key)] + keys.append(as_str_list(key)) elif isinstance(key[0], list): # List of lists of key values; convert to list of list of str - keys = list(map(as_str_list, key)) - elif isinstance(key, str) and len(idx_names) == 1: - # Bare key given for a 1D set; wrap for convenience - keys = [[key]] + keys.extend(map(as_str_list, key)) else: # Other, invalid value raise ValueError(key) - # Process comments to a list of str, or let them all be None - comments = as_str_list(comment) if comment else repeat(None, len(keys)) + if isinstance(comment, str) or comment is None: + comments.append(comment) + else: + # Sequence of comments + comments.extend(comment) + + # Convert a None value into a list of None matching `keys` + if comments == [None]: + comments = comments * len(keys) + + # Elements to send to backend + elements = [] # Combine iterators to tuples. If the lengths are mismatched, the sentinel # value 'False' is filled in - to_add = list(zip_longest(keys, comments, fillvalue=(False,))) - - # Check processed arguments - for e, c in to_add: - # Check for sentinel values - if e == (False,): - raise ValueError(f"Comment {repr(c)} without matching key") + for k, c in list(zip_longest(keys, comments, fillvalue=(False,))): + # Check for sentinel value + if k == (False,): + raise ValueError(f"Comment {c!r} without matching key") elif c == (False,): - raise ValueError(f"Key {repr(e)} without matching comment") - elif len(idx_names) and len(idx_names) != len(e): + raise ValueError(f"Key {k!r} without matching comment") + elif len(idx_names) and len(idx_names) != len(k): raise ValueError( - f"{len(e)}-D key {repr(e)} invalid for " - f"{len(idx_names)}-D set {name}{repr(idx_names)}" + f"{len(k)}-D key {k!r} invalid for {len(idx_names)}-D set " + f"{name}{idx_names!r}" ) + elements.append((k, None, None, c)) + # Send to backend - elements = ((kc[0], None, None, kc[1]) for kc in to_add) self._backend("item_set_elements", "set", name, elements) def remove_set( @@ -515,7 +535,8 @@ def list_items( #: List all defined variables. See :meth:`list_items`. var_list = partialmethod(list_items, ItemType.VAR) - def add_par( + # FIXME reduce complexity 15 → ≤13 + def add_par( # noqa: C901 self, name: str, key_or_data: Optional[Union[str, Sequence[str], Dict, pd.DataFrame]] = None, @@ -569,7 +590,7 @@ def add_par( keys = [keys] # Use the same value for all keys - values = [float(value)] * len(keys) + values: List[Any] = [float(value)] * len(keys) else: # Multiple values values = value diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index 90c2fbf78..3eb6125ea 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -365,16 +365,15 @@ def add_timeseries( df.columns = df.columns.astype(int) # Identify columns to drop - to_drop = set() - if year_lim[0]: - to_drop |= set(filter(lambda y: y < year_lim[0], df.columns)) - if year_lim[1]: - to_drop |= set(filter(lambda y: y > year_lim[1], df.columns)) + def predicate(y: Any) -> bool: + return y < (year_lim[0] or y) or (year_lim[1] or y) < y - df.drop(to_drop, axis=1, inplace=True) + df.drop(list(filter(predicate, df.columns)), axis=1, inplace=True) # Add one time series per row - for (r, v, u, sa), data in df.iterrows(): + for key, data in df.iterrows(): + assert isinstance(key, tuple) + r, v, u, sa = key # Values as float; exclude NA self._backend( "set_data", r, v, data.astype(float).dropna().to_dict(), u, sa, meta @@ -429,19 +428,16 @@ def timeseries( year if isinstance(year, Sequence) else [] if year is None else [year], ), columns=FIELDS["ts_get"], - ) - df["model"] = self.model - df["scenario"] = self.scenario + ).assign(model=self.model, scenario=self.scenario) # drop `subannual` column if not requested (False) or required ('auto') if subannual is not True: has_subannual = not all(df["subannual"] == "Year") if subannual is False and has_subannual: - msg = ( - "timeseries data has subannual values, ", - "use `subannual=True or 'auto'`", + raise ValueError( + "timeseries data has subannual values, use `subannual=True or " + "'auto'`" ) - raise ValueError(msg) if not has_subannual: df.drop("subannual", axis=1, inplace=True) @@ -450,8 +446,11 @@ def timeseries( index = IAMC_IDX if "subannual" in df.columns: index = index + ["subannual"] - df = df.pivot_table(index=index, columns="year")["value"].reset_index() - df.columns.names = [None] + df = ( + df.pivot_table(index=index, columns="year")["value"] + .reset_index() + .rename_axis(columns=None) + ) return df diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index 5a9f93954..5c6f53ac0 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -2,9 +2,9 @@ from itertools import zip_longest from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Set, Union +import genno import pandas as pd import pint -from genno import Quantity from genno.util import parse_units from ixmp.core.timeseries import TimeSeries @@ -14,6 +14,8 @@ from .util import dims_for_qty, get_reversed_rename_dims if TYPE_CHECKING: + from genno.types import AnyQuantity + from ixmp.core.scenario import Scenario log = logging.getLogger(__name__) @@ -25,7 +27,7 @@ def data_for_quantity( column: Literal["mrg", "lvl", "value"], scenario: "Scenario", config: Mapping[str, Mapping], -) -> Quantity: +) -> "AnyQuantity": """Retrieve data from `scenario`. Parameters @@ -47,7 +49,7 @@ def data_for_quantity( Returns ------- - ~genno.Quantity + .Quantity Data for `name`. """ log.debug(f"{name}: retrieve data") @@ -130,14 +132,13 @@ def data_for_quantity( data = data.rename(columns=common.RENAME_DIMS).set_index(dims) # Convert to a Quantity, assign attributes and name - qty = Quantity( + qty = genno.Quantity( data[column], name=name + ("-margin" if column == "mrg" else ""), attrs=attrs ) try: # Remove length-1 dimensions for scalars - # TODO Remove exclusion when genno provides a signature for Quantity.squeeze - qty = qty.squeeze("index", drop=True) # type: ignore [attr-defined] + qty = qty.squeeze("index", drop=True) except (KeyError, ValueError): # KeyError if "index" does not exist; ValueError if its length is > 1 pass @@ -185,8 +186,8 @@ def get_ts( return ts.timeseries(iamc=iamc, subannual=subannual, **filters) -def map_as_qty(set_df: pd.DataFrame, full_set): - """Convert *set_df* to a :class:`~.genno.Quantity`. +def map_as_qty(set_df: pd.DataFrame, full_set) -> "AnyQuantity": + """Convert `set_df` to a :class:`.Quantity`. For the MESSAGE sets named ``cat_*`` (see :ref:`message_ix:mapping-sets`) :meth:`ixmp.Scenario.set` returns a :class:`~pandas.DataFrame` with two columns: @@ -218,7 +219,7 @@ def map_as_qty(set_df: pd.DataFrame, full_set): return ( set_df.set_index([set_from, set_to])["value"] .rename_axis(index=names) - .pipe(Quantity) + .pipe(genno.Quantity) ) @@ -307,13 +308,13 @@ def store_ts(scenario, *data, strict: bool = False) -> None: scenario.commit(f"Data added using {__name__}") -def update_scenario(scenario, *quantities, params=[]): +def update_scenario(scenario, *quantities, params=[]) -> None: """Update *scenario* with computed data from reporting *quantities*. Parameters ---------- scenario : Scenario - quantities : Quantity or pandas.DataFrame + quantities : .Quantity or pandas.DataFrame If DataFrame, must be valid input to :meth:`.Scenario.add_par`. params : list of str, optional For every element of `quantities` that is a pd.DataFrame, the element of diff --git a/ixmp/testing/__init__.py b/ixmp/testing/__init__.py index 8c6fbf68c..1eee1f923 100644 --- a/ixmp/testing/__init__.py +++ b/ixmp/testing/__init__.py @@ -31,6 +31,7 @@ get_cell_output """ + import logging import os import shutil diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 5d85ce84a..877c64f6e 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -3,14 +3,13 @@ from math import ceil from typing import Any, List +import genno import numpy as np import pandas as pd import pint -import xarray as xr from ixmp import Platform, Scenario, TimeSeries from ixmp.backend import IAMC_IDX -from ixmp.report import Quantity #: Common (model name, scenario name) pairs for testing. SCEN = { @@ -145,9 +144,8 @@ def add_test_data(scen: Scenario): # Data ureg = pint.get_application_registry() - x = Quantity( - xr.DataArray(np.random.rand(len(t), len(y)), coords=[("t", t), ("y", y)]), - units=ureg.kg, + x = genno.Quantity( + np.random.rand(len(t), len(y)), coords={"t": t, "y": y}, units=ureg.kg ) # As a pd.DataFrame with units diff --git a/ixmp/testing/jupyter.py b/ixmp/testing/jupyter.py index 6cead889f..3c3cdbf9e 100644 --- a/ixmp/testing/jupyter.py +++ b/ixmp/testing/jupyter.py @@ -1,4 +1,5 @@ """Testing Juypter notebooks.""" + import os import sys from warnings import warn diff --git a/ixmp/testing/resource.py b/ixmp/testing/resource.py index ee9a5f6fb..2806bc16a 100644 --- a/ixmp/testing/resource.py +++ b/ixmp/testing/resource.py @@ -1,4 +1,5 @@ """Performance testing.""" + import logging from collections import namedtuple from typing import Any, Optional diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 10f6ad1e9..bca620420 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -1,7 +1,9 @@ """Tests of :class:`ixmp.Platform`.""" + import logging import re from sys import getrefcount +from typing import TYPE_CHECKING from weakref import getweakrefcount import pandas as pd @@ -13,6 +15,9 @@ from ixmp.backend import FIELDS from ixmp.testing import DATA, assert_logs, models +if TYPE_CHECKING: + from ixmp import Platform + class TestPlatform: def test_init(self): @@ -67,14 +72,11 @@ def test_scenario_list(mp): assert scenario[0] == "Hitchhiker" -def test_export_timeseries_data(test_mp, tmp_path): +def test_export_timeseries_data(mp: "Platform", tmp_path) -> None: path = tmp_path / "export.csv" - test_mp.export_timeseries_data( - path, model="Douglas Adams", unit="???", region="World" - ) + mp.export_timeseries_data(path, model="Douglas Adams", unit="???", region="World") obs = pd.read_csv(path, index_col=False, header=0) - exp = ( DATA[0] .assign(**models["h2g2"], version=1, subannual="Year", meta=0) diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index af07dc44f..c61460a39 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -239,6 +239,23 @@ def test_add_par2(self, scen): scen.init_par("foo", idx_sets=["i", "i", "j"], idx_names=["i0", "i1", "j"]) scen.add_par("foo", pd.DataFrame(columns=["i0", "i1", "j"]), value=1.0) + def test_add_set(self, scen_empty) -> None: + # NB See also test_set(), below + scen = scen_empty + + # Initialize a 0-D set + scen.init_set("i") + scen.init_set("foo", idx_sets=["i"]) + + # Exception raised on invalid arguments + with pytest.raises(ValueError, match="ambiguous; both key.*"): + scen.add_set("i", pd.DataFrame(columns=["i", "comment"]), comment="FOO") + + # Bare str for 1-D set key is wrapped automatically + scen.add_set("i", "i0") + scen.add_set("foo", "i0") + assert {"i0"} == set(scen.set("foo")["i"]) + # Retrieve data def test_idx(self, scen): assert scen.idx_sets("d") == ["i", "j"] @@ -549,7 +566,7 @@ def test_gh_210(scen_empty): assert all(foo_data.columns == columns) -def test_set(scen_empty): +def test_set(scen_empty) -> None: """Test ixmp.Scenario.add_set(), .set(), and .remove_set().""" scen = scen_empty diff --git a/ixmp/tests/report/test_operator.py b/ixmp/tests/report/test_operator.py index 5628d0e38..9e653c3b8 100644 --- a/ixmp/tests/report/test_operator.py +++ b/ixmp/tests/report/test_operator.py @@ -1,11 +1,13 @@ import logging import re from functools import partial +from typing import cast +import genno import pandas as pd import pyam import pytest -from genno import ComputationError, Computer, Quantity +from genno import ComputationError, Computer from genno.testing import assert_qty_equal from pandas.testing import assert_frame_equal @@ -25,7 +27,7 @@ pytestmark = pytest.mark.usefixtures("parametrize_quantity_class") -def test_from_url(test_mp): +def test_from_url(test_mp) -> None: ts = make_dantzig(test_mp) full_url = f"ixmp://{ts.platform.name}/{ts.url}" @@ -43,7 +45,7 @@ def test_from_url(test_mp): assert ts.url == result.url -def test_get_remove_ts(caplog, test_mp): +def test_get_remove_ts(caplog, test_mp) -> None: ts = make_dantzig(test_mp) caplog.set_level(logging.INFO, "ixmp") @@ -79,7 +81,7 @@ def test_get_remove_ts(caplog, test_mp): assert 3 == len(ts.timeseries()) -def test_map_as_qty(): +def test_map_as_qty() -> None: b = ["b1", "b2", "b3", "b4"] input = pd.DataFrame( [["f1", "b1"], ["f1", "b2"], ["f2", "b3"]], columns=["foo", "bar"] @@ -87,7 +89,7 @@ def test_map_as_qty(): result = map_as_qty(input, b) - exp = Quantity( + exp = genno.Quantity( pd.DataFrame( [ ["f1", "b1", 1], @@ -105,7 +107,7 @@ def test_map_as_qty(): assert_qty_equal(exp, result) -def test_update_scenario(caplog, test_mp): +def test_update_scenario(caplog, test_mp) -> None: scen = make_dantzig(test_mp) scen.check_out() scen.add_set("j", "toronto") @@ -122,7 +124,8 @@ def test_update_scenario(caplog, test_mp): c.add("target", scen) # Create a pd.DataFrame suitable for Scenario.add_par() - data = dantzig_data["d"].query("j == 'chicago'").assign(j="toronto") + d = cast(pd.DataFrame, dantzig_data["d"]) + data = d.query("j == 'chicago'").assign(j="toronto") data["value"] += 1.0 # Add to the Reporter @@ -140,11 +143,11 @@ def test_update_scenario(caplog, test_mp): assert len(scen.par("d")) == N_before + len(data) # Modify the data - data = pd.concat([dantzig_data["d"], data]).reset_index(drop=True) + data = pd.concat([d, data]).reset_index(drop=True) data["value"] *= 2.0 # Convert to a Quantity object and re-add - q = Quantity(data.set_index(["i", "j"])["value"], name="d", units="km") + q = genno.Quantity(data.set_index(["i", "j"])["value"], name="d", units="km") c.add("input", q) # Revise the task; the parameter name ('demand') is read from the Quantity @@ -158,7 +161,7 @@ def test_update_scenario(caplog, test_mp): assert_frame_equal(scen.par("d"), data) -def test_store_ts(request, caplog, test_mp): +def test_store_ts(request, caplog, test_mp) -> None: # Computer and target scenario c = Computer() diff --git a/ixmp/tests/report/test_reporter.py b/ixmp/tests/report/test_reporter.py index c612bd349..230f9ad19 100644 --- a/ixmp/tests/report/test_reporter.py +++ b/ixmp/tests/report/test_reporter.py @@ -23,7 +23,7 @@ def scenario(test_mp): @pytest.mark.usefixtures("protect_rename_dims") -def test_configure(test_mp, test_data_path): +def test_configure(test_mp, test_data_path) -> None: # Configure globally; handles 'rename_dims' section configure(rename_dims={"i": "i_renamed"}) @@ -43,7 +43,7 @@ def test_configure(test_mp, test_data_path): pytest.raises(KeyError, rep.get, "i") -def test_reporter_from_scenario(scenario): +def test_reporter_from_scenario(scenario) -> None: r = Reporter.from_scenario(scenario) r.finalize(scenario) @@ -51,7 +51,7 @@ def test_reporter_from_scenario(scenario): assert "scenario" in r.graph -def test_platform_units(test_mp, caplog, ureg): +def test_platform_units(test_mp, caplog, ureg) -> None: """Test handling of units from ixmp.Platform. test_mp is loaded with some units including '-', '???', 'G$', etc. which @@ -142,7 +142,7 @@ def test_platform_units(test_mp, caplog, ureg): assert unit.dimensionality == {"[USD]": 1, "[pkm]": -1} -def test_cli(ixmp_cli, test_mp, test_data_path): +def test_cli(ixmp_cli, test_mp, test_data_path) -> None: # Put something in the database test_mp.open_db() make_dantzig(test_mp) @@ -187,7 +187,7 @@ def test_cli(ixmp_cli, test_mp, test_data_path): ), result.output -def test_filters(test_mp, tmp_path, caplog): +def test_filters(test_mp, tmp_path, caplog) -> None: """Reporting can be filtered ex ante.""" scen = ixmp.Scenario(test_mp, "Reporting filters", "Reporting filters", "new") t, t_foo, t_bar, x = add_test_data(scen) diff --git a/ixmp/tests/report/test_util.py b/ixmp/tests/report/test_util.py index 47ff277ae..a8353bfd9 100644 --- a/ixmp/tests/report/test_util.py +++ b/ixmp/tests/report/test_util.py @@ -1,4 +1,4 @@ -def test_import_rename_dims(): +def test_import_rename_dims() -> None: """RENAME_DIMS can be imported from .report.util, though defined in .common.""" from ixmp.report.util import RENAME_DIMS diff --git a/ixmp/tests/test_compat.py b/ixmp/tests/test_compat.py index 319e575b9..aad7ad94d 100644 --- a/ixmp/tests/test_compat.py +++ b/ixmp/tests/test_compat.py @@ -1,4 +1,5 @@ """Tests for compatibility with other packages, especially :mod:`message_ix`.""" + from pathlib import Path import pytest diff --git a/ixmp/tests/test_perf.py b/ixmp/tests/test_perf.py index 03cee0a66..b7ad54aa8 100644 --- a/ixmp/tests/test_perf.py +++ b/ixmp/tests/test_perf.py @@ -1,4 +1,5 @@ """Performance tests.""" + from functools import partial import pytest diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 2655f45b9..8cf70219a 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -1,4 +1,5 @@ """Tests for ixmp.util.""" + import logging import numpy as np diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index f6c1798a9..34968bd16 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import ( TYPE_CHECKING, - Dict, + Any, Iterable, Iterator, List, @@ -57,7 +57,7 @@ def logger(): return logging.getLogger("ixmp") -def as_str_list(arg, idx_names: Optional[Iterable[str]] = None): +def as_str_list(arg, idx_names: Optional[Iterable[str]] = None) -> List[str]: """Convert various `arg` to list of str. Several types of arguments are handled: @@ -70,11 +70,11 @@ def as_str_list(arg, idx_names: Optional[Iterable[str]] = None): """ if arg is None: - return None + return [] elif idx_names is None: # arg must be iterable - # NB narrower ABC Sequence does not work here; e.g. test_excel_io() - # fails via Scenario.add_set(). + # NB narrower ABC Sequence does not work here; e.g. test_excel_io() fails via + # Scenario.add_set(). if isinstance(arg, Iterable) and not isinstance(arg, str): return list(map(str, arg)) else: @@ -120,7 +120,7 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: ] # State variables for loop name = ["", ""] - data: List[pd.DataFrame] = [None, None] + data: List[pd.DataFrame] = [pd.DataFrame(), pd.DataFrame()] # Elements for first iteration name[0], data[0] = next(items[0]) @@ -146,7 +146,7 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: # Either merge on remaining columns; or, for scalars, on the indices on = sorted(set(left.columns) - {"value", "unit"}) - on_arg: Dict[str, object] = ( + on_arg: Mapping[str, Any] = ( dict(on=on) if on else dict(left_index=True, right_index=True) ) @@ -173,7 +173,7 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: except StopIteration: # No more data for this iterator. # Use "~" because it sorts after all ASCII characters - name[i], data[i] = "~ end", None + name[i], data[i] = "~ end", pd.DataFrame() if name[0] == name[1] == "~ end": break @@ -618,13 +618,8 @@ def _git_log(mod): # Retrieve git log information, if any gl = _git_log(mod) - try: - version = mod.__version__ - except Exception: - # __version__ not available - version = "installed" - finally: - info.append((module_name, version + gl)) + version = getattr(mod, "__version__", "installed") or "" + info.append((module_name, version + gl)) if module_name == "jpype": info.append(("… JVM path", mod.getDefaultJVMPath())) diff --git a/ixmp/util/sphinx_linkcode_github.py b/ixmp/util/sphinx_linkcode_github.py index d3987082e..86cd6209f 100644 --- a/ixmp/util/sphinx_linkcode_github.py +++ b/ixmp/util/sphinx_linkcode_github.py @@ -10,7 +10,7 @@ from sphinx.util import logging -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: import sphinx.application log = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 1ce146a96..421a53896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ documentation = "https://docs.messageix.org/ixmp" [project.optional-dependencies] docs = [ "ixmp[tests]", + "genno >= 1.26", # For sphinx extensions; see doc/conf.py "GitPython", "numpydoc", "sphinx >= 3.0", @@ -83,12 +84,10 @@ omit = ["ixmp/util/sphinx_linkcode_github.py"] [[tool.mypy.overrides]] # Packages/modules for which no type hints are available. module = [ - "dask.*", "jpype", "memory_profiler", - "pandas.*", - "pyam", "pretenders.*", + "pyam", ] ignore_missing_imports = true @@ -107,15 +106,14 @@ markers = [ "performance: ixmp performance test.", ] -[tool.ruff] +[tool.ruff.lint] select = ["C9", "E", "F", "I", "W"] - -[tool.ruff.mccabe] # FIXME the following exceed this limit -# .backend.io.s_read_excel: 26 -# .backend.jdbc.JDBCBackend.item_get_elements: 19 -# .core.scenario.Scenario.add_set: 17 -max-complexity = 15 +# .backend.io.s_read_excel: 22 +# .backend.jdbc.JDBCBackend.item_get_elements: 18 +# .core.scenario.Scenario.add_par: 15 +# .core.scenario.Scenario.add_set: 18 +mccabe.max-complexity = 13 [tool.setuptools.packages] find = {}