From 5a51c2da3bc88c0aa018de91f5fb6a14562f936c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 24 Sep 2024 16:12:44 +0200 Subject: [PATCH 01/13] Update Python versions in "pytest" workflow - Add Python 3.13 - Drop Python 3.8 --- .github/workflows/pytest.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 19f58be70..d835f6d4f 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -24,11 +24,11 @@ jobs: - ubuntu-latest - windows-latest python-version: - - "3.8" # Earliest version supported by ixmp - - "3.9" + - "3.9" # Earliest version supported by ixmp - "3.10" - "3.11" - - "3.12" # Latest supported by ixmp + - "3.12" + - "3.13" # Latest supported by ixmp gams-version: # Version used until 2024-07; disabled # - 25.1.1 @@ -43,17 +43,13 @@ jobs: exclude: # Specific version combinations that are invalid / not to be used - # No arm64 distribution for this version of GAMS - # - { os: macos-latest, gams-version: 25.1.1} # No arm64 distributions of JPype for these Pythons - - { os: macos-latest, python-version: "3.8" } - { os: macos-latest, python-version: "3.9" } # Redundant with macos-latest - { os: macos-13, python-version: "3.10" } - { os: macos-13, python-version: "3.11" } - { os: macos-13, python-version: "3.12" } - # Example: pandas 2.0 requires Python >= 3.8 - # - { python-version: "3.7", pandas-version: "==2.0.0rc0" } + - { os: macos-13, python-version: "3.13" } fail-fast: false @@ -80,6 +76,8 @@ jobs: - uses: r-lib/actions/setup-r@v2 id: setup-r + with: + r-version: "4.4.1" - name: Cache GAMS installer and R packages uses: actions/cache@v4 From 77bd7d5c9c5966e1ed1c922029f0b3495d08fcce Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 24 Sep 2024 16:13:59 +0200 Subject: [PATCH 02/13] Update version classifiers in pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 54e1b867e..2ecdc7f8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,16 +20,16 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: R", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Information Analysis", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "click", "genno >= 1.20", From e7c2942680b00f29e6eb2f414b7a795ee76d306d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 24 Sep 2024 17:02:55 +0200 Subject: [PATCH 03/13] Use standard collections for type hints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python ≥3.9 feature. --- ixmp/_config.py | 10 ++--- ixmp/backend/__init__.py | 9 +++-- ixmp/backend/base.py | 59 ++++++++++++++--------------- ixmp/backend/jdbc.py | 6 +-- ixmp/cli.py | 3 +- ixmp/core/platform.py | 6 +-- ixmp/core/scenario.py | 42 ++++++++------------ ixmp/core/timeseries.py | 8 ++-- ixmp/model/__init__.py | 4 +- ixmp/model/base.py | 4 +- ixmp/report/common.py | 4 +- ixmp/report/operator.py | 4 +- ixmp/report/reporter.py | 4 +- ixmp/testing/data.py | 6 +-- ixmp/tests/backend/test_jdbc.py | 3 +- ixmp/util/__init__.py | 19 +++------- ixmp/util/sphinx_linkcode_github.py | 4 +- 17 files changed, 85 insertions(+), 110 deletions(-) diff --git a/ixmp/_config.py b/ixmp/_config.py index e31ad287a..d5f9f139d 100644 --- a/ixmp/_config.py +++ b/ixmp/_config.py @@ -4,7 +4,7 @@ from copy import copy from dataclasses import asdict, dataclass, field, fields, make_dataclass from pathlib import Path -from typing import Any, Dict, Optional, Tuple, Type +from typing import Any, Optional log = logging.getLogger(__name__) @@ -121,7 +121,7 @@ def delete_field(self, name): data.pop(name) return new_cls, new_cls(**data) - def keys(self) -> Tuple[str, ...]: + def keys(self) -> tuple[str, ...]: return tuple(map(lambda f: f.name.replace("_", " "), fields(self))) def set(self, name: str, value: Any, strict: bool = True): @@ -214,7 +214,7 @@ class Config: #: ``ixmp.config.values["platform"]["platform name"]…``. values: BaseValues - _ValuesClass: Type + _ValuesClass: type[BaseValues] def __init__(self, read: bool = True): self._ValuesClass = BaseValues @@ -261,7 +261,7 @@ def get(self, name: str) -> Any: """Return the value of a configuration key `name`.""" return self.values[name] - def keys(self) -> Tuple[str, ...]: + def keys(self) -> tuple[str, ...]: """Return the names of all registered configuration keys.""" return self.values.keys() @@ -383,7 +383,7 @@ def add_platform(self, name: str, *args, **kwargs): self.values["platform"][name] = info - def get_platform_info(self, name: str) -> Tuple[str, Dict[str, Any]]: + def get_platform_info(self, name: str) -> tuple[str, dict[str, Any]]: """Return information on configured Platform `name`. Parameters diff --git a/ixmp/backend/__init__.py b/ixmp/backend/__init__.py index 4c595f3f7..70c4a5fd0 100644 --- a/ixmp/backend/__init__.py +++ b/ixmp/backend/__init__.py @@ -1,7 +1,10 @@ """Backend API.""" from enum import IntFlag -from typing import Dict, List, Type, Union +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: + import ixmp.backend.base #: Lists of field names for tuples returned by Backend API methods. #: @@ -46,12 +49,12 @@ #: Partial list of dimensions for the IAMC data structure, or “IAMC format”. This omits #: "year" and "subannual" which appear in some variants of the structure, but not in #: others. -IAMC_IDX: List[Union[str, int]] = ["model", "scenario", "region", "variable", "unit"] +IAMC_IDX: list[Union[str, int]] = ["model", "scenario", "region", "variable", "unit"] #: Mapping from names to available backends. To register additional backends, add #: entries to this dictionary. -BACKENDS: Dict[str, Type] = {} +BACKENDS: dict[str, type["ixmp.backend.base.Backend"]] = {} class ItemType(IntFlag): diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index a13dbd946..61775f5dc 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -6,15 +6,12 @@ from pathlib import Path from typing import ( Any, - Dict, Hashable, Iterable, - List, Literal, MutableMapping, Optional, Sequence, - Tuple, Union, ) @@ -33,7 +30,7 @@ class Backend(ABC): # Typing: # - All methods MUST be fully typed. # - Use more permissive types, e.g. Sequence[str], for inputs. - # - Use precise types, e.g. List[str], for return values. + # - Use precise types, e.g. list[str], for return values. # - Backend subclasses do not need to repeat the type annotations; these are implied # by this parent class. # @@ -57,7 +54,7 @@ def __call__(self, obj, method, *args, **kwargs): # Platform methods @classmethod - def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> Dict[str, Any]: + def handle_config(cls, args: Sequence, kwargs: MutableMapping) -> dict[str, Any]: """OPTIONAL: Handle platform/backend config arguments. Returns a :class:`dict` to be stored in the configuration file. This @@ -124,7 +121,7 @@ def set_doc(self, domain: str, docs) -> None: """ @abstractmethod - def get_doc(self, domain: str, name: Optional[str] = None) -> Union[str, Dict]: + def get_doc(self, domain: str, name: Optional[str] = None) -> Union[str, dict]: """Read documentation from database Parameters @@ -155,7 +152,7 @@ def close_db(self) -> None: Close any database connection(s), if open. """ - def get_auth(self, user: str, models: Sequence[str], kind: str) -> Dict[str, bool]: + def get_auth(self, user: str, models: Sequence[str], kind: str) -> dict[str, bool]: """OPTIONAL: Return user authorization for `models`. If the Backend implements access control, this method **must** indicate whether @@ -215,7 +212,7 @@ def set_node( """ @abstractmethod - def get_nodes(self) -> Iterable[Tuple[str, Optional[str], str, str]]: + def get_nodes(self) -> Iterable[tuple[str, Optional[str], str, str]]: """Iterate over all nodes stored on the Platform. Yields @@ -238,7 +235,7 @@ def get_nodes(self) -> Iterable[Tuple[str, Optional[str], str, str]]: """ @abstractmethod - def get_timeslices(self) -> Iterable[Tuple[str, str, float]]: + def get_timeslices(self) -> Iterable[tuple[str, str, float]]: """Iterate over subannual timeslices defined on the Platform instance. Yields @@ -321,7 +318,7 @@ def get_scenario_names(self) -> Iterable[str]: def get_scenarios( self, default: bool, model: Optional[str], scenario: Optional[str] ) -> Iterable[ - Tuple[str, str, str, bool, bool, str, str, str, str, str, str, str, int] + tuple[str, str, str, bool, bool, str, str, str, str, str, str, str, int] ]: """Iterate over TimeSeries stored on the Platform. @@ -377,7 +374,7 @@ def set_unit(self, name: str, comment: str) -> None: """ @abstractmethod - def get_units(self) -> List[str]: + def get_units(self) -> list[str]: """Return all registered symbols for units of measurement. Returns @@ -592,7 +589,7 @@ def preload(self, ts: TimeSeries) -> None: """OPTIONAL: Load `ts` data into memory.""" @staticmethod - def _handle_rw_filters(filters: dict) -> Tuple[Optional[TimeSeries], Dict]: + def _handle_rw_filters(filters: dict) -> tuple[Optional[TimeSeries], dict]: """Helper for :meth:`read_file` and :meth:`write_file`. The `filters` argument is unpacked if the 'scenarios' key is a single @@ -617,7 +614,7 @@ def get_data( variable: Sequence[str], unit: Sequence[str], year: Sequence[str], - ) -> Iterable[Tuple[str, str, str, int, float]]: + ) -> Iterable[tuple[str, str, str, int, float]]: """Retrieve time series data. Parameters @@ -650,7 +647,7 @@ def get_data( @abstractmethod def get_geo( self, ts: TimeSeries - ) -> Iterable[Tuple[str, str, int, str, str, str, bool]]: + ) -> Iterable[tuple[str, str, int, str, str, str, bool]]: """Retrieve time-series 'geodata'. Yields @@ -677,7 +674,7 @@ def set_data( ts: TimeSeries, region: str, variable: str, - data: Dict[int, float], + data: dict[int, float], unit: str, subannual: str, meta: bool, @@ -831,7 +828,7 @@ def has_solution(self, s: Scenario) -> bool: """ @abstractmethod - def list_items(self, s: Scenario, type: str) -> List[str]: + def list_items(self, s: Scenario, type: str) -> list[str]: """Return a list of names of items of `type`. Parameters @@ -882,7 +879,7 @@ def delete_item(self, s: Scenario, type: str, name: str) -> None: """ @abstractmethod - def item_index(self, s: Scenario, name: str, sets_or_names: str) -> List[str]: + def item_index(self, s: Scenario, name: str, sets_or_names: str) -> list[str]: """Return the index sets or names of item `name`. Parameters @@ -900,8 +897,8 @@ def item_get_elements( s: Scenario, type: Literal["equ", "par", "set", "var"], name: str, - filters: Optional[Dict[str, List[Any]]] = None, - ) -> Union[Dict[str, Any], pd.Series, pd.DataFrame]: + filters: Optional[dict[str, list[Any]]] = None, + ) -> Union[dict[str, Any], pd.Series, pd.DataFrame]: """Return elements of item `name`. Parameters @@ -945,7 +942,7 @@ def item_set_elements( s: Scenario, type: str, name: str, - elements: Iterable[Tuple[Any, Optional[float], Optional[str], Optional[str]]], + elements: Iterable[tuple[Any, Optional[float], Optional[str], Optional[str]]], ) -> None: """Add keys or values to item `name`. @@ -1011,7 +1008,7 @@ def get_meta( scenario: Optional[str], version: Optional[int], strict: bool, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """Retrieve all metadata attached to a specific target. Depending on which of `model`, `scenario`, `version` are :obj:`None`, metadata @@ -1121,7 +1118,7 @@ def clear_solution(self, s: Scenario, from_year=None): # Methods for message_ix.Scenario @abstractmethod - def cat_list(self, ms: Scenario, name: str) -> List[str]: + def cat_list(self, ms: Scenario, name: str) -> list[str]: """Return list of categories in mapping `name`. Parameters @@ -1136,7 +1133,7 @@ def cat_list(self, ms: Scenario, name: str) -> List[str]: """ @abstractmethod - def cat_get_elements(self, ms: Scenario, name: str, cat: str) -> List[str]: + def cat_get_elements(self, ms: Scenario, name: str, cat: str) -> list[str]: """Get elements of a category mapping. Parameters @@ -1188,11 +1185,11 @@ class CachingBackend(Backend): #: Cache of values. Keys are given by :meth:`_cache_key`; values depend on the #: subclass' usage of the cache. - _cache: Dict[Tuple, object] = {} + _cache: dict[tuple, object] = {} #: Count of number of times a value was retrieved from cache successfully #: using :meth:`cache_get`. - _cache_hit: Dict[Tuple, int] = {} + _cache_hit: dict[tuple, int] = {} # Backend API methods @@ -1217,8 +1214,8 @@ def _cache_key( ts: TimeSeries, ix_type: Optional[str], name: Optional[str], - filters: Optional[Dict[str, Hashable]] = None, - ) -> Tuple[Hashable, ...]: + filters: Optional[dict[str, Hashable]] = None, + ) -> tuple[Hashable, ...]: """Return a hashable cache key. ixmp `filters` (a :class:`dict` of :class:`list`) are converted to a unique id @@ -1237,7 +1234,7 @@ def _cache_key( return (ts_id, ix_type, name, hash(json.dumps(sorted(filters.items())))) def cache_get( - self, ts: TimeSeries, ix_type: str, name: str, filters: Dict + self, ts: TimeSeries, ix_type: str, name: str, filters: dict ) -> Optional[Any]: """Retrieve value from cache. @@ -1258,7 +1255,7 @@ def cache_get( raise KeyError(ts, ix_type, name, filters) def cache( - self, ts: TimeSeries, ix_type: str, name: str, filters: Dict, value: Any + self, ts: TimeSeries, ix_type: str, name: str, filters: dict, value: Any ) -> bool: """Store `value` in cache. @@ -1284,7 +1281,7 @@ def cache_invalidate( ts: TimeSeries, ix_type: Optional[str] = None, name: Optional[str] = None, - filters: Optional[Dict] = None, + filters: Optional[dict] = None, ) -> None: """Invalidate cached values. @@ -1300,7 +1297,7 @@ def cache_invalidate( if filters is None: i = slice(1) if (ix_type is name is None) else slice(3) - to_remove: Iterable[Tuple] = filter( + to_remove: Iterable[tuple] = filter( lambda k: k[i] == key[i], self._cache.keys() ) else: diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 805b9054b..f562df1c0 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -10,7 +10,7 @@ from functools import lru_cache from pathlib import Path, PurePosixPath from types import SimpleNamespace -from typing import Generator, List, Mapping, Optional +from typing import Generator, Mapping, Optional from weakref import WeakKeyDictionary import jpype @@ -152,7 +152,7 @@ def _handle_jexception(): @lru_cache -def _fixed_index_sets(scheme: str) -> Mapping[str, List[str]]: +def _fixed_index_sets(scheme: str) -> Mapping[str, list[str]]: """Return index sets for items that are fixed in the Java code. See :meth:`JDBCBackend.init_item`. The return value is cached so the method is only @@ -1290,7 +1290,7 @@ def start_jvm(jvmargs=None): # Conversion methods -def to_pylist(jlist) -> List: +def to_pylist(jlist) -> list: """Convert Java list types to :class:`list`.""" try: return list(jlist[:]) diff --git a/ixmp/cli.py b/ixmp/cli.py index 0a0169624..d4649d28b 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -1,11 +1,10 @@ from pathlib import Path -from typing import Type import click import ixmp -ScenarioClass: Type[ixmp.Scenario] = ixmp.Scenario +ScenarioClass: type[ixmp.Scenario] = ixmp.Scenario class VersionType(click.ParamType): diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index ce7021858..bf9de6786 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -1,6 +1,6 @@ import logging from os import PathLike -from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union +from typing import TYPE_CHECKING, Optional, Sequence, Union import numpy as np import pandas as pd @@ -264,7 +264,7 @@ def add_unit(self, unit: str, comment: str = "None") -> None: self._backend.set_unit(unit, comment) - def units(self) -> List[str]: + def units(self) -> list[str]: """Return all units defined on the Platform. Returns @@ -386,7 +386,7 @@ def add_timeslice(self, name: str, category: str, duration: float) -> None: def check_access( self, user: str, models: Union[str, Sequence[str]], access: str = "view" - ) -> Union[bool, Dict[str, bool]]: + ) -> Union[bool, dict[str, bool]]: """Check access to specific models. Parameters diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 26de09b20..b7480c504 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -4,17 +4,7 @@ from numbers import Real from os import PathLike from pathlib import Path -from typing import ( - Any, - Callable, - Dict, - Iterable, - List, - MutableSequence, - Optional, - Sequence, - Union, -) +from typing import Any, Callable, Iterable, MutableSequence, Optional, Sequence, Union from warnings import warn import pandas as pd @@ -135,7 +125,7 @@ def load_scenario_data(self) -> None: for name in getattr(self, "{}_list".format(ix_type))(): get_func(name) - def idx_sets(self, name: str) -> List[str]: + def idx_sets(self, name: str) -> list[str]: """Return the list of index sets for an item (set, par, var, equ). Parameters @@ -145,7 +135,7 @@ def idx_sets(self, name: str) -> List[str]: """ return self._backend("item_index", name, "sets") - def idx_names(self, name: str) -> List[str]: + def idx_names(self, name: str) -> list[str]: """Return the list of index names for an item (set, par, var, equ). Parameters @@ -167,8 +157,8 @@ def _keys(self, name, key_or_keys): return [str(key_or_keys)] def set( - self, name: str, filters: Optional[Dict[str, Sequence[str]]] = None, **kwargs - ) -> Union[List[str], pd.DataFrame]: + self, name: str, filters: Optional[dict[str, Sequence[str]]] = None, **kwargs + ) -> Union[list[str], pd.DataFrame]: """Return the (filtered) elements of a set. Parameters @@ -190,7 +180,7 @@ def set( def add_set( # noqa: C901 self, name: str, - key: Union[str, Sequence[str], Dict, pd.DataFrame], + key: Union[str, Sequence[str], dict, pd.DataFrame], comment: Union[str, Sequence[str], None] = None, ) -> None: """Add elements to an existing set. @@ -229,7 +219,7 @@ def add_set( # noqa: C901 # List of keys keys: MutableSequence[Union[str, MutableSequence[str]]] = [] # List of comments for each key - comments: List[Optional[str]] = [] + comments: list[Optional[str]] = [] # Check arguments and convert to two lists: keys and comments if len(idx_names) == 0: @@ -311,7 +301,7 @@ def add_set( # noqa: C901 def remove_set( self, name: str, - key: Optional[Union[str, Sequence[str], Dict, pd.DataFrame]] = None, + key: Optional[Union[str, Sequence[str], dict, pd.DataFrame]] = None, ) -> None: """Delete set elements or an entire set. @@ -329,7 +319,7 @@ def remove_set( self._backend("item_delete_elements", "set", name, self._keys(name, key)) def par( - self, name: str, filters: Optional[Dict[str, Sequence[str]]] = None, **kwargs + self, name: str, filters: Optional[dict[str, Sequence[str]]] = None, **kwargs ) -> pd.DataFrame: """Return parameter data. @@ -354,7 +344,7 @@ def par( def items( self, type: ItemType = ItemType.PAR, - filters: Optional[Dict[str, Sequence[str]]] = None, + filters: Optional[dict[str, Sequence[str]]] = None, *, indexed_by: Optional[str] = None, par_data: Optional[bool] = None, @@ -517,7 +507,7 @@ def init_item( def list_items( self, item_type: ItemType, indexed_by: Optional[str] = None - ) -> List[str]: + ) -> list[str]: """List all defined items of type `item_type`. See also @@ -539,7 +529,7 @@ def list_items( def add_par( # noqa: C901 self, name: str, - key_or_data: Optional[Union[str, Sequence[str], Dict, pd.DataFrame]] = None, + key_or_data: Optional[Union[str, Sequence[str], dict, pd.DataFrame]] = None, value=None, unit: Optional[str] = None, comment: Optional[str] = None, @@ -590,7 +580,7 @@ def add_par( # noqa: C901 keys = [keys] # Use the same value for all keys - values: List[Any] = [float(value)] * len(keys) + values: list[Any] = [float(value)] * len(keys) else: # Multiple values values = value @@ -661,7 +651,7 @@ def init_scalar(self, name: str, val: Real, unit: str, comment=None) -> None: self.init_par(name, [], []) self.change_scalar(name, val, unit, comment) - def scalar(self, name: str) -> Dict[str, Union[Real, str]]: + def scalar(self, name: str) -> dict[str, Union[Real, str]]: """Return the value and unit of a scalar. Parameters @@ -824,7 +814,7 @@ def solve( self, model: Optional[str] = None, callback: Optional[Callable] = None, - cb_kwargs: Dict[str, Any] = {}, + cb_kwargs: dict[str, Any] = {}, **model_options, ) -> None: """Solve the model and store output. @@ -924,7 +914,7 @@ def to_excel( self, path: PathLike, items: ItemType = ItemType.SET | ItemType.PAR, - filters: Optional[Dict[str, Union[Sequence[str], "Scenario"]]] = None, + filters: Optional[dict[str, Union[Sequence[str], "Scenario"]]] = None, max_row: Optional[int] = None, ) -> None: """Write Scenario to a Microsoft Excel file. diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index 3eb6125ea..a1af105f6 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -2,7 +2,7 @@ from contextlib import contextmanager, nullcontext from os import PathLike from pathlib import Path -from typing import Any, Dict, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Literal, Optional, Sequence, Union from warnings import warn from weakref import ProxyType, proxy @@ -117,7 +117,7 @@ def __del__(self): @classmethod def from_url( cls, url: str, errors: Literal["warn", "raise"] = "warn" - ) -> Tuple[Optional["TimeSeries"], Platform]: + ) -> tuple[Optional["TimeSeries"], Platform]: """Instantiate a TimeSeries (or Scenario) given an ``ixmp://`` URL. The following are equivalent:: @@ -303,7 +303,7 @@ def add_timeseries( self, df: pd.DataFrame, meta: bool = False, - year_lim: Tuple[Optional[int], Optional[int]] = (None, None), + year_lim: tuple[Optional[int], Optional[int]] = (None, None), ) -> None: """Add time series data. @@ -562,7 +562,7 @@ def get_meta(self, name: Optional[str] = None): ) return all_meta[name] if name else all_meta - def set_meta(self, name_or_dict: Union[str, Dict[str, Any]], value=None) -> None: + def set_meta(self, name_or_dict: Union[str, dict[str, Any]], value=None) -> None: """Set :ref:`data-meta` for this object. Parameters diff --git a/ixmp/model/__init__.py b/ixmp/model/__init__.py index 16a7daac9..cc8b89077 100644 --- a/ixmp/model/__init__.py +++ b/ixmp/model/__init__.py @@ -1,8 +1,6 @@ -from typing import Dict, Type - #: Mapping from names to available models. To register additional models, #: add elements to this variable. -MODELS: Dict[str, Type] = {} +MODELS: dict[str, type] = {} def get_model(name, **model_options): diff --git a/ixmp/model/base.py b/ixmp/model/base.py index 5332fbfd2..417b6d4b6 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -2,7 +2,7 @@ import os import re from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Dict, Mapping +from typing import TYPE_CHECKING, Mapping from ixmp.util import maybe_check_out, maybe_commit @@ -85,7 +85,7 @@ def initialize(cls, scenario): log.debug(f"No initialization for {repr(scenario.scheme)}-scheme Scenario") @classmethod - def initialize_items(cls, scenario: "Scenario", items: Mapping[str, Dict]) -> None: + def initialize_items(cls, scenario: "Scenario", items: Mapping[str, dict]) -> None: """Helper for :meth:`initialize`. All of the `items` are added to `scenario`. Existing items are not modified. diff --git a/ixmp/report/common.py b/ixmp/report/common.py index 97f3b0efe..b1211f889 100644 --- a/ixmp/report/common.py +++ b/ixmp/report/common.py @@ -1,5 +1,3 @@ -from typing import Dict - #: Dimensions to rename when extracting raw data from Scenario objects. #: Mapping from Scenario dimension name -> preferred dimension name. -RENAME_DIMS: Dict[str, str] = {} +RENAME_DIMS: dict[str, str] = {} diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index 5c6f53ac0..f508bebfb 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -1,6 +1,6 @@ import logging from itertools import zip_longest -from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Set, Union +from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Union import genno import pandas as pd @@ -147,7 +147,7 @@ def data_for_quantity( # Non-weak references to objects to keep them alive -_FROM_URL_REF: Set[Any] = set() +_FROM_URL_REF: set[Any] = set() def from_url(url: str, cls=TimeSeries) -> "TimeSeries": diff --git a/ixmp/report/reporter.py b/ixmp/report/reporter.py index 3c7fcfd1e..3eff09dd4 100644 --- a/ixmp/report/reporter.py +++ b/ixmp/report/reporter.py @@ -1,5 +1,5 @@ from itertools import chain, repeat -from typing import List, Union, cast +from typing import Union, cast import dask import pandas as pd @@ -50,7 +50,7 @@ def from_scenario(cls, scenario: Scenario, **kwargs) -> "Reporter": rep.add("scenario", scenario) # List of top-level keys - all_keys: List[Union[str, Key]] = [] + all_keys: list[Union[str, Key]] = [] # List of parameters, equations, and variables quantities = chain( diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 9de18009c..8f551bf67 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -1,7 +1,7 @@ # Methods are in alphabetical order from itertools import product from math import ceil -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional import genno import numpy as np @@ -36,13 +36,13 @@ class ScenarioKwargs(TypedDict, total=False): #: Common (model name, scenario name) pairs for testing. -SCEN: Dict[str, "ScenarioIdentifiers"] = { +SCEN: dict[str, "ScenarioIdentifiers"] = { "dantzig": dict(model="canning problem", scenario="standard"), "h2g2": dict(model="Douglas Adams", scenario="Hitchhiker"), } models = SCEN -_MS: List[Any] = [models["dantzig"]["model"], models["dantzig"]["scenario"]] +_MS: list[Any] = [models["dantzig"]["model"], models["dantzig"]["scenario"]] #: Time series data for testing. HIST_DF = pd.DataFrame( diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index aebfc56bb..efc765c27 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -3,7 +3,6 @@ import os import platform from sys import getrefcount -from typing import Tuple import jpype import numpy as np @@ -296,7 +295,7 @@ def test_cache_arg(arg, request): # This variable formerly had 'warns' as the third element in some tuples, to # test for deprecation warnings. -INIT_PARAMS: Tuple[Tuple, ...] = ( +INIT_PARAMS: tuple[tuple, ...] = ( # Handled in JDBCBackend: ( ["nonexistent.properties"], diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index 34968bd16..fc3e221e6 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -7,16 +7,7 @@ from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.util import find_spec from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Iterable, - Iterator, - List, - Mapping, - Optional, - Tuple, -) +from typing import TYPE_CHECKING, Any, Iterable, Iterator, Mapping, Optional from urllib.parse import urlparse from warnings import warn @@ -57,7 +48,7 @@ def logger(): return logging.getLogger("ixmp") -def as_str_list(arg, idx_names: Optional[Iterable[str]] = None) -> List[str]: +def as_str_list(arg, idx_names: Optional[Iterable[str]] = None) -> list[str]: """Convert various `arg` to list of str. Several types of arguments are handled: @@ -100,7 +91,7 @@ def check_year(y, s): return True -def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: +def diff(a, b, filters=None) -> Iterator[tuple[str, pd.DataFrame]]: """Compute the difference between Scenarios `a` and `b`. :func:`diff` combines :func:`pandas.merge` and :meth:`.Scenario.items`. Only @@ -111,7 +102,7 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: Yields ------ tuple of str, pandas.DataFrame - Tuples of item name and data. + tuples of item name and data. """ # Iterators; index 0 corresponds to `a`, 1 to `b` items = [ @@ -120,7 +111,7 @@ def diff(a, b, filters=None) -> Iterator[Tuple[str, pd.DataFrame]]: ] # State variables for loop name = ["", ""] - data: List[pd.DataFrame] = [pd.DataFrame(), pd.DataFrame()] + data: list[pd.DataFrame] = [pd.DataFrame(), pd.DataFrame()] # Elements for first iteration name[0], data[0] = next(items[0]) diff --git a/ixmp/util/sphinx_linkcode_github.py b/ixmp/util/sphinx_linkcode_github.py index 86cd6209f..8279a2ceb 100644 --- a/ixmp/util/sphinx_linkcode_github.py +++ b/ixmp/util/sphinx_linkcode_github.py @@ -6,7 +6,7 @@ from functools import _lru_cache_wrapper, lru_cache, partial from pathlib import Path from types import FunctionType -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from sphinx.util import logging @@ -153,7 +153,7 @@ def linkcode_resolve(self, domain: str, info: dict) -> Optional[str]: try: # Use the info for the first of `candidates` available - line_info: Tuple[str, int, int] = next( + line_info: tuple[str, int, int] = next( filter(None, map(self.line_numbers.get, candidates)) ) except StopIteration: From 1209b35381a3aa7b4ba4a17e6477f050e1182cf1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 20 Nov 2024 11:37:41 +0100 Subject: [PATCH 04/13] Import certain types from collections.abc Import of these from `typing` is deprecated from Python 3.9. --- ixmp/backend/base.py | 12 ++---------- ixmp/backend/jdbc.py | 4 ++-- ixmp/core/platform.py | 3 ++- ixmp/core/scenario.py | 3 ++- ixmp/core/timeseries.py | 3 ++- ixmp/model/base.py | 3 ++- ixmp/model/gams.py | 3 ++- ixmp/report/operator.py | 3 ++- ixmp/util/__init__.py | 3 ++- 9 files changed, 18 insertions(+), 19 deletions(-) diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index 61775f5dc..fd8ab0e54 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -1,19 +1,11 @@ import json import logging from abc import ABC, abstractmethod +from collections.abc import Hashable, Iterable, MutableMapping, Sequence from copy import copy from os import PathLike from pathlib import Path -from typing import ( - Any, - Hashable, - Iterable, - Literal, - MutableMapping, - Optional, - Sequence, - Union, -) +from typing import Any, Literal, Optional, Union import pandas as pd diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index f562df1c0..5c4e83ad8 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -4,13 +4,13 @@ import platform import re from collections import ChainMap -from collections.abc import Iterable, Sequence +from collections.abc import Generator, Iterable, Mapping, Sequence from contextlib import contextmanager from copy import copy from functools import lru_cache from pathlib import Path, PurePosixPath from types import SimpleNamespace -from typing import Generator, Mapping, Optional +from typing import Optional from weakref import WeakKeyDictionary import jpype diff --git a/ixmp/core/platform.py b/ixmp/core/platform.py index bf9de6786..7107773e0 100644 --- a/ixmp/core/platform.py +++ b/ixmp/core/platform.py @@ -1,6 +1,7 @@ import logging +from collections.abc import Sequence from os import PathLike -from typing import TYPE_CHECKING, Optional, Sequence, Union +from typing import TYPE_CHECKING, Optional, Union import numpy as np import pandas as pd diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index b7480c504..fd81ef2e2 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -1,10 +1,11 @@ import logging +from collections.abc import Callable, Iterable, MutableSequence, Sequence from functools import partialmethod from itertools import zip_longest from numbers import Real from os import PathLike from pathlib import Path -from typing import Any, Callable, Iterable, MutableSequence, Optional, Sequence, Union +from typing import Any, Optional, Union from warnings import warn import pandas as pd diff --git a/ixmp/core/timeseries.py b/ixmp/core/timeseries.py index a1af105f6..f80492656 100644 --- a/ixmp/core/timeseries.py +++ b/ixmp/core/timeseries.py @@ -1,8 +1,9 @@ import logging +from collections.abc import Sequence from contextlib import contextmanager, nullcontext from os import PathLike from pathlib import Path -from typing import Any, Literal, Optional, Sequence, Union +from typing import Any, Literal, Optional, Union from warnings import warn from weakref import ProxyType, proxy diff --git a/ixmp/model/base.py b/ixmp/model/base.py index 417b6d4b6..33af65c09 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -2,7 +2,8 @@ import os import re from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Mapping +from collections.abc import Mapping +from typing import TYPE_CHECKING from ixmp.util import maybe_check_out, maybe_commit diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 20d8eaf35..819516319 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -3,11 +3,12 @@ import re import shutil import tempfile +from collections.abc import MutableMapping from copy import copy from pathlib import Path from subprocess import CalledProcessError, check_output, run from tempfile import TemporaryDirectory -from typing import Any, MutableMapping, Optional +from typing import Any, Optional from ixmp.backend import ItemType from ixmp.model.base import Model, ModelError diff --git a/ixmp/report/operator.py b/ixmp/report/operator.py index f508bebfb..b0cca010f 100644 --- a/ixmp/report/operator.py +++ b/ixmp/report/operator.py @@ -1,6 +1,7 @@ import logging +from collections.abc import Mapping from itertools import zip_longest -from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Union +from typing import TYPE_CHECKING, Any, Literal, Optional, Union import genno import pandas as pd diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index fc3e221e6..91f143dfe 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -1,13 +1,14 @@ import logging import re import sys +from collections.abc import Iterable, Iterator, Mapping from contextlib import contextmanager from functools import lru_cache from importlib.abc import MetaPathFinder from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.util import find_spec from pathlib import Path -from typing import TYPE_CHECKING, Any, Iterable, Iterator, Mapping, Optional +from typing import TYPE_CHECKING, Any, Optional from urllib.parse import urlparse from warnings import warn From fcb4cd3c04bad3731e08fde12aca084c2dfa69db Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Mon, 28 Oct 2024 15:49:41 +0100 Subject: [PATCH 05/13] Update Python versions in install instructions --- doc/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install.rst b/doc/install.rst index fa4de9680..07cd2268a 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -34,7 +34,7 @@ Install system dependencies Python ------ -Python version 3.8 or later is required. +Python version 3.9 or later is required. GAMS (required) --------------- @@ -89,7 +89,7 @@ After installing GAMS, we recommend that new users install Anaconda, and then us Advanced users may choose to install :mod:`ixmp` from source code (next section). 4. Install Python via either `Miniconda`_ or `Anaconda`_. [1]_ - We recommend the latest version; currently Python 3.10. [2]_ + We recommend the latest version; currently Python 3.13. [2]_ 5. Open a command prompt. We recommend Windows users use the “Anaconda Prompt” to avoid issues with permissions and environment variables when installing and using :mod:`ixmp`. From dc0cc58212d89c393eb62d854497b5eae2e670b1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 8 Nov 2024 10:49:01 +0100 Subject: [PATCH 06/13] Gitignore .vscode --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1ab84214a..f420afdb3 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ prof/ # Editors and IDEs .editorconfig +.vscode *.iml /**/.idea From 026097a1755137388bf3d6b063c02ad0e935790f Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 19 Nov 2024 08:34:39 +0100 Subject: [PATCH 07/13] Adjust two tests for more informative exceptions --- ixmp/tests/core/test_scenario.py | 10 ++-------- ixmp/tests/test_cli.py | 8 +------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index 6259066fb..3592280f0 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -1,5 +1,4 @@ import re -import sys from pathlib import Path from shutil import copyfile @@ -98,13 +97,8 @@ def test_from_url(self, mp, caplog): # Giving an invalid scenario with errors='warn' causes a message to be logged msg = ( - "ValueError: " - + ( - "scenario='Hitchhikerfoo'" - if sys.version_info.minor != 12 - else "model, scenario, or version not found" - ) - + f"\nwhen loading Scenario from url: {repr(url + 'foo')}" + "ValueError: scenario='Hitchhikerfoo'\n" + f"when loading Scenario from url: {repr(url + 'foo')}" ) with assert_logs(caplog, msg): scen, mp = ixmp.Scenario.from_url(url + "foo") diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index 15c66db1d..982414319 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -1,5 +1,4 @@ import re -import sys from pathlib import Path import pandas as pd @@ -422,12 +421,7 @@ def test_solve(ixmp_cli, test_mp): ] result = ixmp_cli.invoke(cmd) assert result.exit_code == 1, result.output - exp = ( - "='non-existing'" - if sys.version_info.minor != 12 - else ", scenario, or version not found" - ) - assert f"Error: model{exp}" in result.output + assert "Error: model='non-existing'" in result.output result = ixmp_cli.invoke([f"--url=ixmp://{test_mp.name}/foo/bar", "solve"]) assert UsageError.exit_code == result.exit_code, result.output From d5f9ef06caf0843816b616d36beaf874396b0661 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 19 Nov 2024 15:52:09 +0100 Subject: [PATCH 08/13] Replace pretenders with pytest-httpserver in tests - Simpliy test_access.py, fixtures, and data. - Update [tests] dependencies in pyproject.toml - Update mypy config and pre-commit environment --- .pre-commit-config.yaml | 1 + ...cess.properties => test_access.properties} | 0 .../test_check_single_model_access.properties | 16 -- ixmp/tests/test_access.py | 140 ++++++------------ pyproject.toml | 8 +- 5 files changed, 51 insertions(+), 114 deletions(-) rename ixmp/tests/data/{test_check_multi_model_access.properties => test_access.properties} (100%) delete mode 100644 ixmp/tests/data/test_check_single_model_access.properties diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 05ffac9f4..89b28ca82 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: - pandas-stubs - pytest - sphinx + - werkzeug - xarray args: ["."] - repo: https://github.com/astral-sh/ruff-pre-commit diff --git a/ixmp/tests/data/test_check_multi_model_access.properties b/ixmp/tests/data/test_access.properties similarity index 100% rename from ixmp/tests/data/test_check_multi_model_access.properties rename to ixmp/tests/data/test_access.properties diff --git a/ixmp/tests/data/test_check_single_model_access.properties b/ixmp/tests/data/test_check_single_model_access.properties deleted file mode 100644 index 24a595ba8..000000000 --- a/ixmp/tests/data/test_check_single_model_access.properties +++ /dev/null @@ -1,16 +0,0 @@ -# Used by test_access.py - -config.name = unit_test_db@local - -jdbc.driver = org.hsqldb.jdbcDriver -jdbc.url = jdbc:hsqldb:mem:test_access -jdbc.user = ixmp -jdbc.pwd = ixmp - -application.tag = IXSE_SR15 -application.serverURL = http://localhost:8888 - -config.server.url = {auth_url} -config.server.config = DemoDB -config.server.username = service_user_dev -config.server.password = service_user_dev diff --git a/ixmp/tests/test_access.py b/ixmp/tests/test_access.py index 42d6df993..bd24be44d 100644 --- a/ixmp/tests/test_access.py +++ b/ixmp/tests/test_access.py @@ -1,116 +1,64 @@ -import logging -import sys -from subprocess import Popen -from time import sleep +import json import pytest -from pretenders.client.http import HTTPMock -from pretenders.common.constants import FOREVER import ixmp from ixmp.testing import create_test_platform -log = logging.getLogger(__name__) +@pytest.fixture +def mock(httpserver): + """Mock server with responses for both tests.""" + from werkzeug import Request, Response -@pytest.fixture(scope="session") -def server(): - proc = Popen( - [ - sys.executable, - "-m", - "pretenders.server.server", - "--host", - "localhost", - "--port", - "8000", - ] + httpserver.expect_request("/login", method="POST").respond_with_json( + "security-token" ) - log.info(f"Mock server started with pid {proc.pid}") - - # Wait for server to start up - sleep(5) - yield + # Mock the behaviour of the ixmp_source (Java) access API + # - Request data is valid JSON containing a list of dict. + # - Response is a JSON list of bool of equal length. + def handler(r: Request) -> Response: + data = r.get_json() + result = [ + (i["username"], i["entityType"], i["entityId"]) + == ("test_user", "MODEL", "test_model") + for i in data + ] + return Response(json.dumps(result), content_type="application/json") - proc.terminate() - log.info("Mock server terminated") + # Use the same handler for all test requests against the /access/list URL + httpserver.expect_request( + "/access/list", + method="POST", + headers={"Authorization": "Bearer security-token"}, # JSON Web Token header + ).respond_with_handler(handler) + return httpserver -@pytest.fixture(scope="function") -def mock(server): - # Create the mock server - httpmock = HTTPMock("localhost", 8000) - # Common responses for both tests - httpmock.when("POST /login").reply( - '"security-token"', headers={"Content-Type": "application/json"}, times=FOREVER +@pytest.fixture +def test_props(mock, request, tmp_path, test_data_path): + return create_test_platform( + tmp_path, test_data_path, "test_access", auth_url=mock.url_for("") ) - yield httpmock +M = ["test_model", "non_existing_model"] -def test_check_single_model_access(mock, tmp_path, test_data_path, request): - mock.when( - "POST /access/list", - body='.+"test_user".+', - headers={"Authorization": "Bearer security-token"}, - ).reply("[true]", headers={"Content-Type": "application/json"}, times=FOREVER) - mock.when( - "POST /access/list", - body='.+"non_granted_user".+', - headers={"Authorization": "Bearer security-token"}, - ).reply("[false]", headers={"Content-Type": "application/json"}, times=FOREVER) - - test_props = create_test_platform( - tmp_path, - test_data_path, - f"{request.node.name}", - auth_url=mock.pretend_url, - ) +@pytest.mark.parametrize( + "user, models, exp", + ( + ("test_user", "test_model", True), + ("non_granted_user", "test_model", False), + ("non_existing_user", "test_model", False), + ("test_user", M, {"test_model": True, "non_existing_model": False}), + ("non_granted_user", M, {"test_model": False, "non_existing_model": False}), + ("non_existing_user", M, {"test_model": False, "non_existing_model": False}), + ), +) +def test_check_access(test_props, user, models, exp): + """:meth:`.check_access` correctly handles certain arguments and responses.""" mp = ixmp.Platform(backend="jdbc", dbprops=test_props) - - granted = mp.check_access("test_user", "test_model") - assert granted - - granted = mp.check_access("non_granted_user", "test_model") - assert not granted - - granted = mp.check_access("non_existing_user", "test_model") - assert not granted - - -def test_check_multi_model_access(mock, tmp_path, test_data_path, request): - mock.when( - "POST /access/list", - body='.+"test_user".+', - headers={"Authorization": "Bearer security-token"}, - ).reply( - "[true, false]", headers={"Content-Type": "application/json"}, times=FOREVER - ) - mock.when( - "POST /access/list", - body='.+"non_granted_user".+', - headers={"Authorization": "Bearer security-token"}, - ).reply( - "[false, false]", headers={"Content-Type": "application/json"}, times=FOREVER - ) - - test_props = create_test_platform( - tmp_path, test_data_path, f"{request.node.name}", auth_url=mock.pretend_url - ) - - mp = ixmp.Platform(backend="jdbc", dbprops=test_props) - - access = mp.check_access("test_user", ["test_model", "non_existing_model"]) - assert access["test_model"] - assert not access["non_existing_model"] - - access = mp.check_access("non_granted_user", ["test_model", "non_existing_model"]) - assert not access["test_model"] - assert not access["non_existing_model"] - - access = mp.check_access("non_existing_user", ["test_model", "non_existing_model"]) - assert not access["test_model"] - assert not access["non_existing_model"] + assert exp == mp.check_access(user, models) diff --git a/pyproject.toml b/pyproject.toml index 2ecdc7f8c..ede4e147c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,10 +63,10 @@ tests = [ "ixmp[report,tutorial]", "memory_profiler", "nbclient >= 0.5", - "pretenders >= 1.4.4", "pytest >= 5", "pytest-benchmark", "pytest-cov", + "pytest-httpserver", "pytest-rerunfailures", "pytest-xdist", ] @@ -81,12 +81,16 @@ exclude_also = [ ] omit = ["ixmp/util/sphinx_linkcode_github.py"] +[tool.mypy] +exclude = [ + "build/", +] + [[tool.mypy.overrides]] # Packages/modules for which no type hints are available. module = [ "jpype", "memory_profiler", - "pretenders.*", "pyam", ] ignore_missing_imports = true From f0744004c31eed72e8427849d0abcb2fed899110 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 20 Nov 2024 08:57:16 +0100 Subject: [PATCH 09/13] Remove outdated pretenders workaround --- .github/workflows/pytest.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index d835f6d4f..d2dd03870 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -111,9 +111,6 @@ jobs: # see https://github.com/iiasa/ixmp/issues/534 pip install "pint != 0.24.0" "numpy < 2" - # TEMPORARY Work around pretenders/pretenders#153 - pip install "bottle < 0.13" - - name: Install R dependencies and tutorial requirements run: | install.packages(c("remotes", "Rcpp")) From 82a512099cfc1b7d627517127c47efc7340f7aeb Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 20 Nov 2024 08:58:02 +0100 Subject: [PATCH 10/13] Remove outdated pint/numpy workaround --- .github/workflows/pytest.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index d2dd03870..a08c6ef0c 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -107,10 +107,6 @@ jobs: # commented: use with "pandas-version" in the matrix, above # pip install --upgrade pandas${{ matrix.pandas-version }} - # TEMPORARY Work around hgrecco/pint#2007; - # see https://github.com/iiasa/ixmp/issues/534 - pip install "pint != 0.24.0" "numpy < 2" - - name: Install R dependencies and tutorial requirements run: | install.packages(c("remotes", "Rcpp")) From 3ab408afbf63f1edc29d2efdbe61da235c4f6fa9 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 20 Nov 2024 10:37:06 +0100 Subject: [PATCH 11/13] Address pandas apply include_groups warning --- ixmp/report/util.py | 7 ++----- ixmp/util/__init__.py | 13 +++++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ixmp/report/util.py b/ixmp/report/util.py index 3e718cc80..3a2430933 100644 --- a/ixmp/report/util.py +++ b/ixmp/report/util.py @@ -14,11 +14,8 @@ def dims_for_qty(data): :data:`.RENAME_DIMS` is used to rename dimensions. """ - if isinstance(data, pd.DataFrame): - # List of the dimensions - dims = data.columns.tolist() - else: - dims = list(data) + # List of the dimensions + dims = data.columns.tolist() if isinstance(data, pd.DataFrame) else list(data) # Remove columns containing values or units; dimensions are the remainder for col in "value", "lvl", "mrg", "unit": diff --git a/ixmp/util/__init__.py b/ixmp/util/__init__.py index 91f143dfe..35c60fa7e 100644 --- a/ixmp/util/__init__.py +++ b/ixmp/util/__init__.py @@ -497,14 +497,15 @@ def describe(df): info = ( platform.scenario_list(model=model, scen=scenario, default=default_only) .groupby(["model", "scenario"], group_keys=True) - .apply(describe) + .apply(describe, include_groups=False) ) - if len(info): - info = info.reset_index() - else: - # No results; re-create a minimal empty data frame - info = pd.DataFrame([], columns=["model", "scenario", "default", "N"]) + # If we have no results; re-create a minimal empty data frame + info = ( + info.reset_index() + if len(info) + else pd.DataFrame([], columns=["model", "scenario", "default", "N"]) + ) info["scenario"] = info["scenario"].str.cat(info["default"].astype(str), sep="#") From 5f087ec776457ac5b8d457067c3bfed60778631c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 20 Nov 2024 11:23:18 +0100 Subject: [PATCH 12/13] Override pyam-iamc pin to old Pint on Python 3.13 --- .github/workflows/pytest.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index a08c6ef0c..b2fc14a3a 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -107,6 +107,10 @@ jobs: # commented: use with "pandas-version" in the matrix, above # pip install --upgrade pandas${{ matrix.pandas-version }} + # TEMPORARY With Python 3.13 pyam-iamc resolves to 1.3.1, which in turn + # limits pint < 0.17. Override. cf. iiasa/ixmp#544 + pip install --upgrade pint + - name: Install R dependencies and tutorial requirements run: | install.packages(c("remotes", "Rcpp")) From dcf15afee265ddc3e33aa9bd473e4d82864f525b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 24 Sep 2024 16:26:41 +0200 Subject: [PATCH 13/13] Add #544 to release notes --- RELEASE_NOTES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 46b14e471..32c4feba8 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -1,6 +1,8 @@ Next release ============ +- :mod:`ixmp` is tested and compatible with `Python 3.13 `__ (:pull:`544`). +- Support for Python 3.8 is dropped (:pull:`544`), as it has reached end-of-life. - :mod:`ixmp` locates GAMS API libraries needed for the Java code underlying :class:`.JDBCBackend` based on the system GAMS installation (:pull:`532`). As a result: