From 3beb956783fb35ad6a1629504d4b9d26467693ad Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 12:48:34 +0100 Subject: [PATCH 01/14] Partly revert #464 (CI workaround for #463) --- .github/workflows/pytest.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 4822261cc..304b989f2 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -104,10 +104,6 @@ jobs: # commented: use with "pandas-version" in the matrix, above # pip install --upgrade pandas${{ matrix.pandas-version }} - - name: TEMPORARY Work around iiasa/ixmp#463 - if: matrix.python-version != '3.11' - run: pip install "JPype1 != 1.4.1" - - name: Install R dependencies and tutorial requirements run: | install.packages(c("remotes", "Rcpp")) From f00ebabd2deead77f399c462d7f39d4b27bab95a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 12:53:25 +0100 Subject: [PATCH 02/14] Support Python 3.12 (closes #501) --- .github/workflows/pytest.yaml | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 304b989f2..657fd7564 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -28,8 +28,8 @@ jobs: - "3.8" # Earliest version supported by ixmp - "3.9" - "3.10" - - "3.11" # Latest supported by ixmp - # - "3.12" # Pending JPype support; see iiasa/ixmp#501 + - "3.11" + - "3.12" # Latest supported by ixmp # commented: force a specific version of pandas, for e.g. pre-release # testing diff --git a/pyproject.toml b/pyproject.toml index 8b573c047..d66fea7f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: R", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Information Analysis", From 9e15da974cf68fca7a41ea98fbd13017acfe9950 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 13:06:26 +0100 Subject: [PATCH 03/14] Adjust ABC tests for Python 3.12 exception text --- ixmp/tests/backend/test_base.py | 4 +++- ixmp/tests/test_model.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index 77fc1a91f..dd9a90db5 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -69,7 +69,9 @@ class BE2(Backend): def test_class(): # An incomplete Backend subclass can't be instantiated with pytest.raises( - TypeError, match="Can't instantiate abstract class BE1 with abstract methods" + TypeError, + match="Can't instantiate abstract class BE1 with(out an implementation for)? " + "abstract methods", ): BE1() diff --git a/ixmp/tests/test_model.py b/ixmp/tests/test_model.py index 8187decff..21c22da78 100644 --- a/ixmp/tests/test_model.py +++ b/ixmp/tests/test_model.py @@ -16,7 +16,9 @@ class M1(Model): pass with pytest.raises( - TypeError, match="Can't instantiate abstract class M1 " "with abstract methods" + TypeError, + match="Can't instantiate abstract class M1 with(out an implementation for)? " + "abstract methods", ): M1() From ef632e809e7271f6df95f8f6b2602918077494e9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 13:08:18 +0100 Subject: [PATCH 04/14] Remove workaround for #494 --- .github/workflows/pytest.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 657fd7564..1ed1493c3 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -92,10 +92,6 @@ jobs: run: echo "RETICULATE_PYTHON=$pythonLocation" >> $GITHUB_ENV shell: bash - - name: Work around https://bugs.launchpad.net/lxml/+bug/2035206 - if: matrix.python-version == '3.8' && matrix.os == 'macos-latest' - run: pip install "lxml == 4.9.2" - - name: Install Python package and dependencies # [docs] contains [tests], which contains [report,tutorial] run: | From eb93f42c497f77d967b4d2e027ea97357b689221 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 13:12:14 +0100 Subject: [PATCH 05/14] Remove unused isort config --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d66fea7f5..44ff031b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,11 +78,6 @@ exclude_also = [ ] omit = ["ixmp/util/sphinx_linkcode_github.py"] -[tool.isort] -profile = "black" - -[tool.mypy] - [[tool.mypy.overrides]] # Packages/modules for which no type hints are available. module = [ From 3632b8f9c0aeec5aba8c3d6adfdfc1c4cf81f6ee Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 13:18:20 +0100 Subject: [PATCH 06/14] Address test_diff_data() warnings with pandas 2.1 --- ixmp/tests/test_util.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ixmp/tests/test_util.py b/ixmp/tests/test_util.py index 12c71a694..2655f45b9 100644 --- a/ixmp/tests/test_util.py +++ b/ixmp/tests/test_util.py @@ -92,17 +92,17 @@ def test_diff_data(test_mp): # Expected results exp_b = pd.DataFrame( [ - ["chicago", 300.0, "cases", np.NaN, None, "left_only"], - ["new-york", np.NaN, None, 325.0, "cases", "right_only"], + ["chicago", 300.0, "cases", np.nan, np.nan, "left_only"], + ["new-york", np.nan, np.nan, 325.0, "cases", "right_only"], ["topeka", 275.0, "cases", 275.0, "cases", "both"], ], columns="j value_a unit_a value_b unit_b _merge".split(), ) exp_d = pd.DataFrame( [ - ["san-diego", "chicago", np.NaN, None, 1.8, "km", "right_only"], - ["san-diego", "new-york", np.NaN, None, 2.5, "km", "right_only"], - ["san-diego", "topeka", np.NaN, None, 1.4, "km", "right_only"], + ["san-diego", "chicago", np.nan, np.nan, 1.8, "km", "right_only"], + ["san-diego", "new-york", np.nan, np.nan, 2.5, "km", "right_only"], + ["san-diego", "topeka", np.nan, np.nan, 1.4, "km", "right_only"], ["seattle", "chicago", 1.7, "km", 123.4, "km", "both"], ["seattle", "new-york", 2.5, "km", 123.4, "km", "both"], ["seattle", "topeka", 1.8, "km", 123.4, "km", "both"], From 9af3f4f2ba57df9e4d3dc37c22bdd4e80f7561fe Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 14:42:30 +0100 Subject: [PATCH 07/14] Handle SystemError in JDBCBackend.get() Occurs only with Python 3.12 and JPype 1.5.0. --- ixmp/backend/jdbc.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 637783bd1..4d998d3ae 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -301,7 +301,7 @@ def __init__(self, jvmargs=None, **kwargs): f"{e}\nCheck that dependencies of ixmp.jar are " f"included in {Path(__file__).parents[2] / 'lib'}" ) - except jpype.JException as e: # pragma: no cover + except java.Exception as e: # pragma: no cover # Handle Java exceptions jclass = e.__class__.__name__ if jclass.endswith("HikariPool.PoolInitializationException"): @@ -708,10 +708,18 @@ def get(self, ts): # either getTimeSeries or getScenario method = getattr(self.jobj, "get" + ts.__class__.__name__) - # Re-raise as a ValueError for bad model or scenario name, or other - with _handle_jexception(): + # Re-raise as a ValueError for bad model or scenario name, or other with + # with _handle_jexception(): + try: # Either the 2- or 3- argument form, depending on args jobj = method(*args) + except SystemError: + # JPype 1.5.0 with Python 3.12: " returned a result with an exception set" + # At least transmute to a ValueError + raise ValueError("model, scenario, or version not found") + except BaseException as e: + _raise_jexception(e) self._index_and_set_attrs(jobj, ts) @@ -938,7 +946,7 @@ def init_item(self, s, type, name, idx_sets, idx_names): # by Backend, so don't return here try: func(name, idx_sets, idx_names) - except jpype.JException as e: + except java.Exception as e: if "already exists" in e.args[0]: raise ValueError(f"{repr(name)} already exists") else: From b4a135ebf585a592c9214c86eb5f2f5f89b0b963 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 14:43:38 +0100 Subject: [PATCH 08/14] Adjust two tests for less-informative exceptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …with Python 3.12 and JPype 1.5.0 only. --- ixmp/tests/core/test_scenario.py | 12 +++++++++--- ixmp/tests/test_cli.py | 8 +++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index 886ed73f3..2deed0cba 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -1,4 +1,5 @@ import re +import sys import numpy.testing as npt import pandas as pd @@ -93,10 +94,15 @@ def test_from_url(self, mp, caplog): with pytest.raises(Exception, match=expected): scen, mp = ixmp.Scenario.from_url(url + "#10000", errors="raise") - # Giving an invalid scenario with errors='warn' raises an exception + # Giving an invalid scenario with errors='warn' causes a message to be logged msg = ( - "ValueError: scenario='Hitchhikerfoo'\n" - f"when loading Scenario from url: {repr(url + 'foo')}" + "ValueError: " + + ( + "scenario='Hitchhikerfoo'" + if sys.version_info.minor != 12 + else "model, scenario, or version not found" + ) + + f"\nwhen loading Scenario from url: {repr(url + 'foo')}" ) with assert_logs(caplog, msg): scen, mp = ixmp.Scenario.from_url(url + "foo") diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index be641955a..48dd48151 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -1,4 +1,5 @@ import re +import sys from pathlib import Path import pandas as pd @@ -403,7 +404,12 @@ def test_solve(ixmp_cli, test_mp): ] result = ixmp_cli.invoke(cmd) assert result.exit_code == 1, result.output - assert "Error: model='non-existing'" in result.output + exp = ( + "='non-existing'" + if sys.version_info.minor != 12 + else ", scenario, or version not found" + ) + assert f"Error: model{exp}" in result.output result = ixmp_cli.invoke([f"--url=ixmp://{test_mp.name}/foo/bar", "solve"]) assert UsageError.exit_code == result.exit_code, result.output From 660e2d8b8f672efbce451663c85c11b664806f61 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 23:30:38 +0100 Subject: [PATCH 09/14] Work around #463 in test suite - Xfail one test, adjust another. --- ixmp/tests/backend/test_base.py | 7 +++++++ ixmp/tests/backend/test_jdbc.py | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index dd9a90db5..4020d9066 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -1,3 +1,4 @@ +import sys from pathlib import Path import pytest @@ -147,6 +148,12 @@ def test_del_ts(self, test_mp): # Cache size has increased assert cache_size_pre + 1 == len(backend._cache) + # JPype ≥ 1.4.1 with Python ≤ 3.10 produces danging traceback/frame references + # to `s` that prevent it being GC'd at "del s" below. See + # https://github.com/iiasa/ixmp/issues/463 and test_jdbc.test_del_ts + if sys.version_info.minor <= 10: + s.__del__() # Force deletion of cached objects associated with `s` + # Delete the object; associated cache is freed del s diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index b17147ef6..f05aa0431 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -2,6 +2,7 @@ import logging import os import platform +import sys from sys import getrefcount from typing import Tuple @@ -370,6 +371,12 @@ def test_verbose_exception(test_mp, exception_verbose_true): assert "at.ac.iiasa.ixmp.Platform.getScenario" in exc_msg +@pytest.mark.xfail( + condition=sys.version_info.minor <= 10, + raises=AssertionError, + # See also test_base.TestCachingBackend.test_del_ts + reason="https://github.com/iiasa/ixmp/issues/463", +) def test_del_ts(): mp = ixmp.Platform( backend="jdbc", From 8ada04ef239f6002a98a9d7ea6c76cdc7969075e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 8 Jan 2024 23:31:34 +0100 Subject: [PATCH 10/14] Use assignment expression in _raise_jexception() --- ixmp/backend/jdbc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 4d998d3ae..892e150ea 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -127,8 +127,7 @@ def _raise_jexception(exc, msg="unhandled Java exception: "): """Convert Java/JPype exceptions to ordinary Python RuntimeError.""" # Try to re-raise as a ValueError for bad model or scenario name arg = exc.args[0] if isinstance(exc.args[0], str) else "" - match = re.search(r"getting '([^']*)' in table '([^']*)'", arg) - if match: + if match := re.search(r"getting '([^']*)' in table '([^']*)'", arg): param = match.group(2).lower() if param in {"model", "scenario"}: raise ValueError(f"{param}={repr(match.group(1))}") from None From 1e29864ee9363cb9232f9f22bf01e865e59f48bf Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 9 Jan 2024 09:37:35 +0100 Subject: [PATCH 11/14] Adapt scenario.add_par to raise instead of xpass --- ixmp/core/scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ixmp/core/scenario.py b/ixmp/core/scenario.py index 1572974ce..5201e634f 100644 --- a/ixmp/core/scenario.py +++ b/ixmp/core/scenario.py @@ -574,7 +574,7 @@ def add_par( # Multiple values values = value - data = pd.DataFrame(zip(keys, values), columns=["key", "value"]) + data = pd.DataFrame(zip_longest(keys, values), columns=["key", "value"]) if data.isna().any(axis=None): raise ValueError("Length mismatch between keys and values") From fd21c96da4272423fff889f9738a2733ececb716 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Tue, 9 Jan 2024 11:36:24 +0100 Subject: [PATCH 12/14] Specify working versions of JPype1 --- RELEASE_NOTES.rst | 3 +++ pyproject.toml | 1 + 2 files changed, 4 insertions(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index aae2f0f90..16b857f29 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -28,6 +28,9 @@ All changes - When a :class:`.GAMSModel` is solved with an LP status of 5 (optimal, but with infeasibilities after unscaling), :class:`.JDBCBackend` would attempt to read the output GDX file and fail, leading to an uninformative error message (:issue:`98`). Now :class:`.ModelError` is raised describing the situation. - Improved type hinting for static typing of code that uses :mod:`ixmp` (:issue:`465`, :pull:`500`). +- :mod:`ixmp` requires on JPype1 1.4.0 or earlier, for Python 3.10 and earlier (:pull:`504`). + With JPype1 1.4.1 and later, memory management in :class:`.CachingBackend` may not function as intended (:issue:`463`), which could lead to high memory use where many, large :class:`.Scenario` objects are created and used in a single Python program. + (For Python 3.11 and later, any version of JPype1 from the prior minimum (1.2.1) to the latest is supported.) .. _v3.7.0: diff --git a/pyproject.toml b/pyproject.toml index 44ff031b2..9cdd9662b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "click", "genno >= 1.16", "JPype1 >= 1.2.1", + "JPype1 <= 1.4.0; python_version < '3.11'", "openpyxl", "pandas >= 1.2", "pint", From 1db4b84ba5504b08fc0e0332e63640c6895e650a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 9 Jan 2024 15:06:11 +0100 Subject: [PATCH 13/14] Add pytest-xdist and use in "pytest" CI workflow --- .github/workflows/pytest.yaml | 8 +++++++- pyproject.toml | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 1ed1493c3..636458566 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -117,7 +117,13 @@ jobs: shell: Rscript {0} - name: Run test suite using pytest - run: pytest ixmp -m "not performance" --verbose -rA --cov-report=xml --color=yes + run: | + pytest ixmp \ + -m "not performance" \ + --color=yes -rA --verbose \ + --cov-report=xml \ + --numprocesses=auto + shell: bash - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v3 diff --git a/pyproject.toml b/pyproject.toml index 9cdd9662b..951af5da2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ tests = [ "pytest-benchmark", "pytest-cov", "pytest-rerunfailures", + "pytest-xdist", ] [project.scripts] From 4f123822ce6ce0ef172a498d25b06ed8087d7e84 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 9 Jan 2024 15:06:33 +0100 Subject: [PATCH 14/14] Add #504 to release notes --- RELEASE_NOTES.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 16b857f29..583017540 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -14,7 +14,8 @@ Code that imports from the old locations will continue to work, but will raise : All changes ----------- -- Support for Python 3.7 is dropped (:pull:`492`). +- :mod:`ixmp` is tested and compatible with `Python 3.12 `__ (:pull:`504`). +- Support for Python 3.7 is dropped (:pull:`492`), as it has reached end-of-life. - Rename :mod:`ixmp.report` and :mod:`ixmp.util` (:pull:`500`). - New reporting operators :func:`.from_url`, :func:`.get_ts`, and :func:`.remove_ts` (:pull:`500`). - New CLI command :program:`ixmp platform copy` and :doc:`CLI documentation ` (:pull:`500`). @@ -28,7 +29,7 @@ All changes - When a :class:`.GAMSModel` is solved with an LP status of 5 (optimal, but with infeasibilities after unscaling), :class:`.JDBCBackend` would attempt to read the output GDX file and fail, leading to an uninformative error message (:issue:`98`). Now :class:`.ModelError` is raised describing the situation. - Improved type hinting for static typing of code that uses :mod:`ixmp` (:issue:`465`, :pull:`500`). -- :mod:`ixmp` requires on JPype1 1.4.0 or earlier, for Python 3.10 and earlier (:pull:`504`). +- :mod:`ixmp` requires JPype1 1.4.0 or earlier, for Python 3.10 and earlier (:pull:`504`). With JPype1 1.4.1 and later, memory management in :class:`.CachingBackend` may not function as intended (:issue:`463`), which could lead to high memory use where many, large :class:`.Scenario` objects are created and used in a single Python program. (For Python 3.11 and later, any version of JPype1 from the prior minimum (1.2.1) to the latest is supported.)