From bdcae9a86d924d3c6866d4fd4ddbaf7ec2ddefb0 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 11:41:01 +0100 Subject: [PATCH 01/21] Add Model.initialize, Model.initialize_items --- ixmp/model/base.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/ixmp/model/base.py b/ixmp/model/base.py index 0ce98c817..00d9c2a1b 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -18,6 +18,60 @@ def __init__(self, name, **kwargs): """ pass + @classmethod + def initialize(cls, scenario): + """Set up *scenario* with required items. + + Parameters + ---------- + scenario : .Scenario + Scenario object to initialize. + + See also + -------- + initialize_items + """ + pass + + @classmethod + def initialize_items(cls, scenario, items): + """Helper for :meth:`initialize`. + + All of the *items* are added to *scenario*. Existing items are not + modified. + + Parameters + ---------- + scenario : .Scenario + Scenario object to initialize. + items : list of dict + Each entry is one ixmp item (set, parameter, equation, or variable) + to initialize. Each dict **must** have the key 'ix_type'; one of + 'set', 'par', 'equ', or 'var'; any other entries are keyword + arguments to the methods :meth:`.init_set` etc. + + See also + -------- + .init_equ + .init_par + .init_set + .init_var + """ + for item_info in items: + # Copy so that pop() below does not modify *items* + item_info = item_info.copy() + + # Get the appropriate method, e.g. init_set or init_par + ix_type = item_info.pop('ix_type') + init_method = getattr(scenario, 'init_{}'.format(ix_type)) + + try: + # Add the item + init_method(**item_info) + except ValueError: + # Item already exists + pass + @abstractmethod def run(self, scenario): """Execute the model. From b0487a32ab11313a21bada1e0e65db74834407f7 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:02:44 +0100 Subject: [PATCH 02/21] Update Model documentation --- doc/source/api-model.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/api-model.rst b/doc/source/api-model.rst index 17cf1cad9..70d4f7c94 100644 --- a/doc/source/api-model.rst +++ b/doc/source/api-model.rst @@ -23,9 +23,9 @@ Model API --------- .. autoclass:: ixmp.model.base.Model - :members: name, __init__, run + :members: name, __init__, initialize, initialize_items, run - In the following, the words REQUIRED, OPTIONAL, etc. have specific meanings as described in `IETF RFC 2119 `_. + In the following, the words **required**, **optional**, etc. have specific meanings as described in `IETF RFC 2119 `_. Model is an **abstract** class; this means it MUST be subclassed. It has two REQURIED methods that MUST be overridden by subclasses: @@ -33,4 +33,6 @@ Model API .. autosummary:: name __init__ + initialize + initialize_items run From 2d9df3e5d4f5167d9d3c4f39246ffa7d2fb813c9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:18:16 +0100 Subject: [PATCH 03/21] Add ixmp.model.dantzig --- ixmp/model/dantzig.py | 60 +++++++++++++++++++++++++++++++++++++++++++ ixmp/testing.py | 48 +++------------------------------- 2 files changed, 63 insertions(+), 45 deletions(-) create mode 100644 ixmp/model/dantzig.py diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py new file mode 100644 index 000000000..18b75e075 --- /dev/null +++ b/ixmp/model/dantzig.py @@ -0,0 +1,60 @@ +import pandas as pd + +from .gams import GAMSModel + + +ITEMS = ( + # Plants + dict(ix_type='set', name='i'), + # Markets + dict(ix_type='set', name='j'), + # Capacity of plant i in cases + dict(ix_type='par', name='a', idx_sets=['i']), + # Demand at market j in cases + dict(ix_type='par', name='b', idx_sets=['j']), + # Distance between plant i and market j + dict(ix_type='par', name='d', idx_sets=['i', 'j']), + # Transport cost per case per 1000 miles + dict(ix_type='par', name='f', idx_sets=None), + # Decision variables and equations + dict(ix_type='var', name='x', idx_sets=['i', 'j']), + dict(ix_type='var', name='z', idx_sets=None), + dict(ix_type='equ', name='cost', idx_sets=None), + dict(ix_type='equ', name='demand', idx_sets=['j']), + dict(ix_type='equ', name='supply', idx_sets=['i']), +) + + +class DantzigModel(GAMSModel): + @classmethod + def initialize(cls, scenario, with_data=False): + # Initialize the ixmp items + cls.initialize_items(scenario, ITEMS) + + if not with_data: + return + + # Add set elements + scenario.add_set('i', ['seattle', 'san-diego']) + scenario.add_set('j', ['new-york', 'chicago', 'topeka']) + + # Add parameter values + scenario.add_par('a', 'seattle', 350, 'cases') + scenario.add_par('a', 'san-diego', 600, 'cases') + + scenario.add_par('b', pd.DataFrame([ + ['new-york', 325, 'cases'], + ['chicago', 300, 'cases'], + ['topeka', 275, 'cases'], + ], columns='j value unit'.split())) + + scenario.add_par('d', pd.DataFrame([ + ['seattle', 'new-york', 2.5, 'km'], + ['seattle', 'chicago', 1.7, 'km'], + ['seattle', 'topeka', 1.8, 'km'], + ['san-diego', 'new-york', 2.5, 'km'], + ['san-diego', 'chicago', 1.8, 'km'], + ['san-diego', 'topeka', 1.4, 'km'], + ], columns='i j value unit'.split())) + + scenario.change_scalar('f', 90.0, 'USD_per_km') diff --git a/ixmp/testing.py b/ixmp/testing.py index c2fd6120f..743246313 100644 --- a/ixmp/testing.py +++ b/ixmp/testing.py @@ -23,6 +23,7 @@ from . import cli, config as ixmp_config from .core import Platform, Scenario, IAMC_IDX +from .model.dantzig import DantzigModel models = { @@ -183,51 +184,8 @@ def make_dantzig(mp, solve=False): annot = "Dantzig's transportation problem for illustration and testing" scen = Scenario(mp, version='new', annotation=annot, **models['dantzig']) - # define sets - scen.init_set('i') - scen.add_set('i', ['seattle', 'san-diego']) - scen.init_set('j') - scen.add_set('j', ['new-york', 'chicago', 'topeka']) - - # capacity of plant i in cases - # add parameter elements one-by-one (string and value) - scen.init_par('a', idx_sets='i') - scen.add_par('a', 'seattle', 350, 'cases') - scen.add_par('a', 'san-diego', 600, 'cases') - - # demand at market j in cases - # add parameter elements as dataframe (with index names) - scen.init_par('b', idx_sets='j') - b_data = pd.DataFrame([ - ['new-york', 325, 'cases'], - ['chicago', 300, 'cases'], - ['topeka', 275, 'cases'], - ], columns=['j', 'value', 'unit']) - scen.add_par('b', b_data) - - # distance in thousands of miles - # add parameter elements as dataframe (with index names) - scen.init_par('d', idx_sets=['i', 'j']) - d_data = pd.DataFrame([ - ['seattle', 'new-york', 2.5, 'km'], - ['seattle', 'chicago', 1.7, 'km'], - ['seattle', 'topeka', 1.8, 'km'], - ['san-diego', 'new-york', 2.5, 'km'], - ['san-diego', 'chicago', 1.8, 'km'], - ['san-diego', 'topeka', 1.4, 'km'], - ], columns='i j value unit'.split()) - scen.add_par('d', d_data) - - # cost per case per 1000 miles - # initialize scalar with a value and a unit - scen.init_scalar('f', 90.0, 'USD_per_km') - - # initialize the decision variables and equations - scen.init_var('x', idx_sets=['i', 'j']) - scen.init_var('z', None, None) - scen.init_equ('cost') - scen.init_equ('demand', idx_sets=['j']) - scen.init_equ('supply', idx_sets=['i']) + # Use the model class' initalize() method to populate the Scenario + DantzigModel.initialize(scen, with_data=True) # commit the scenario scen.commit("Import Dantzig's transport problem for testing.") From 6c65b292613bb80085fd4dec3cff157f0587e931 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:28:03 +0100 Subject: [PATCH 04/21] Call Model.initialize through Scenario.__init__ --- ixmp/__init__.py | 2 ++ ixmp/core.py | 11 ++++++++++- ixmp/testing.py | 10 ++++------ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index be48c9e6b..1b3d5aa95 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -10,6 +10,7 @@ from .backend.jdbc import JDBCBackend from .model import MODELS from .model.gams import GAMSModel +from .model.dantzig import DantzigModel from ixmp.reporting import Reporter # noqa: F401 __version__ = get_versions()['version'] @@ -22,4 +23,5 @@ MODELS.update({ 'default': GAMSModel, 'gams': GAMSModel, + 'dantzig': DantzigModel, }) diff --git a/ixmp/core.py b/ixmp/core.py index a2b0621a6..551171c6b 100644 --- a/ixmp/core.py +++ b/ixmp/core.py @@ -611,7 +611,7 @@ class Scenario(TimeSeries): scheme = None def __init__(self, mp, model, scenario, version=None, scheme=None, - annotation=None, cache=False): + annotation=None, cache=False, **model_init_args): if not isinstance(mp, Platform): raise ValueError('mp is not a valid `ixmp.Platform` instance') @@ -635,6 +635,15 @@ def __init__(self, mp, model, scenario, version=None, scheme=None, def _cache(self): return hasattr(self.platform._backend, '_cache') + # Retrieve the Model class correlating to the *scheme* + try: + model_class = get_model(scheme).__class__ + except KeyError: + pass + else: + # Use the model class to initialize the Scenario + model_class.initialize(self, **model_init_args) + @classmethod def from_url(cls, url, errors='warn'): """Instantiate a Scenario given an ixmp-scheme URL. diff --git a/ixmp/testing.py b/ixmp/testing.py index 743246313..e1662cc8c 100644 --- a/ixmp/testing.py +++ b/ixmp/testing.py @@ -23,7 +23,6 @@ from . import cli, config as ixmp_config from .core import Platform, Scenario, IAMC_IDX -from .model.dantzig import DantzigModel models = { @@ -180,12 +179,11 @@ def make_dantzig(mp, solve=False): pass mp.add_region('DantzigLand', 'country') - # initialize a new (empty) instance of an `ixmp.Scenario` + # Initialize a new Scenario, and use the DantzigModel class' initialize() + # method to populate it annot = "Dantzig's transportation problem for illustration and testing" - scen = Scenario(mp, version='new', annotation=annot, **models['dantzig']) - - # Use the model class' initalize() method to populate the Scenario - DantzigModel.initialize(scen, with_data=True) + scen = Scenario(mp, **models['dantzig'], version='new', annotation=annot, + scheme='dantzig', with_data=True) # commit the scenario scen.commit("Import Dantzig's transport problem for testing.") From 342b0b7165d4bf5985a0a65f5cef13c37a0a1d94 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:35:40 +0100 Subject: [PATCH 05/21] Add DantzigModel to documentation --- doc/source/api-model.rst | 9 +++++++++ ixmp/model/dantzig.py | 10 ++++++++++ ixmp/testing.py | 4 ++++ 3 files changed, 23 insertions(+) diff --git a/doc/source/api-model.rst b/doc/source/api-model.rst index 70d4f7c94..ebbb78c5c 100644 --- a/doc/source/api-model.rst +++ b/doc/source/api-model.rst @@ -15,13 +15,22 @@ Provided models .. automodule:: ixmp.model :members: get_model, MODELS +.. currentmodule:: ixmp.model.gams + .. autoclass:: ixmp.model.gams.GAMSModel :members: +.. currentmodule:: ixmp.model.dantzig + +.. autoclass:: ixmp.model.dantzig.DantzigModel + :members: + Model API --------- +.. currentmodule:: ixmp.model.base + .. autoclass:: ixmp.model.base.Model :members: name, __init__, initialize, initialize_items, run diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index 18b75e075..80f82690a 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -26,8 +26,18 @@ class DantzigModel(GAMSModel): + """Dantzig's cannery/transport problem as a :class:`GAMSModel`. + + Provided for testing :mod:`ixmp` code. + """ @classmethod def initialize(cls, scenario, with_data=False): + """Initialize the problem. + + If *with_data* is :obj:`True` (default: :obj:`False`), the set and + parameter values from the original problem are also populated. + Otherwise, the sets and parameters are left empty. + """ # Initialize the ixmp items cls.initialize_items(scenario, ITEMS) diff --git a/ixmp/testing.py b/ixmp/testing.py index e1662cc8c..1fb5a7770 100644 --- a/ixmp/testing.py +++ b/ixmp/testing.py @@ -170,6 +170,10 @@ def make_dantzig(mp, solve=False): If not :obj:`False`, then *solve* is interpreted as a path to a directory, and the model ``transport_ixmp.gms`` in the directory is run for the scenario. + + See also + -------- + .DantzigModel """ # add custom units and region for timeseries data try: From 551f5816f997942337319cfef41a8f5adae32097 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:54:25 +0100 Subject: [PATCH 06/21] Move Dantzig GAMS file to ixmp/model/ --- .../data/transport_ixmp.gms => ixmp/model/dantzig.gms | 0 ixmp/model/dantzig.py | 10 ++++++++++ ixmp/testing.py | 3 +-- tests/test_core.py | 5 ++--- tests/test_model.py | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) rename tests/data/transport_ixmp.gms => ixmp/model/dantzig.gms (100%) diff --git a/tests/data/transport_ixmp.gms b/ixmp/model/dantzig.gms similarity index 100% rename from tests/data/transport_ixmp.gms rename to ixmp/model/dantzig.gms diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index 80f82690a..4f6f5ea1d 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -1,3 +1,6 @@ +from collections import ChainMap +from pathlib import Path + import pandas as pd from .gams import GAMSModel @@ -30,6 +33,13 @@ class DantzigModel(GAMSModel): Provided for testing :mod:`ixmp` code. """ + name = 'dantzig' + + defaults = ChainMap({ + # Override keys from GAMSModel + 'model_file': Path(__file__).with_name('dantzig.gms'), + }, GAMSModel.defaults) + @classmethod def initialize(cls, scenario, with_data=False): """Initialize the problem. diff --git a/ixmp/testing.py b/ixmp/testing.py index 1fb5a7770..25d8dfe38 100644 --- a/ixmp/testing.py +++ b/ixmp/testing.py @@ -197,8 +197,7 @@ def make_dantzig(mp, solve=False): if solve: # Solve the model using the GAMS code provided in the `tests` folder - scen.solve(model=str(Path(solve) / 'transport_ixmp'), - case='transport_standard') + scen.solve(model='dantzig', case='transport_standard') # add timeseries data for testing `clone(keep_solution=False)` # and `remove_solution()` diff --git a/tests/test_core.py b/tests/test_core.py index d05b59388..08b5ff56b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -428,7 +428,7 @@ def test_log_level_raises(test_mp): pytest.raises(ValueError, test_mp.set_log_level, level='foo') -def test_solve_callback(test_mp, test_data_path): +def test_solve_callback(test_mp): """Test the callback argument to Scenario.solve(). In real usage, callback() would compute some kind of convergence criterion. @@ -441,8 +441,7 @@ def test_solve_callback(test_mp, test_data_path): scen = make_dantzig(test_mp) # Solve the scenario as configured - solve_args = dict(model=str(test_data_path / 'transport_ixmp'), - case='transport_standard', gams_args=['LogOption=2']) + solve_args = dict(model='dantzig', gams_args=['LogOption=2']) scen.solve(**solve_args) # Store the expected value of the decision variable, x diff --git a/tests/test_model.py b/tests/test_model.py index 8053df39c..ea6f415df 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -9,4 +9,4 @@ ], ids=['null-comment', 'null-list', 'empty-list']) def test_GAMSModel(test_mp, test_data_path, kwargs): s = make_dantzig(test_mp) - s.solve(test_data_path / 'transport_ixmp', **kwargs) + s.solve(model='dantzig', **kwargs) From 5306fb49d5040d42583b184a5aa0c243facab7ab Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:55:23 +0100 Subject: [PATCH 07/21] Simplify existing tests - test_data_path need no longer be passed to make_dantzig() --- tests/test_integration.py | 14 +++++++------- tests/test_reporting.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index 81254849c..cb9ec91bc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -10,7 +10,7 @@ TS_DF_CLEARED.loc[0, 2005] = np.nan -def test_run_clone(test_mp, test_data_path, caplog): +def test_run_clone(test_mp, caplog): # this test is designed to cover the full functionality of the GAMS API # - initialize a new platform instance # - creates a new scenario and exports a gdx file @@ -18,7 +18,7 @@ def test_run_clone(test_mp, test_data_path, caplog): # - reads back the solution from the output # - performs the test on the objective value and the timeseries data mp = test_mp - scen = make_dantzig(mp, solve=test_data_path) + scen = make_dantzig(mp, solve=True) assert np.isclose(scen.var('z')['lvl'], 153.675) assert_frame_equal(scen.timeseries(iamc=True), TS_DF) @@ -51,15 +51,15 @@ def test_run_clone(test_mp, test_data_path, caplog): assert_frame_equal(scen4.timeseries(iamc=True), TS_DF_CLEARED) -def test_run_remove_solution(test_mp, test_data_path): +def test_run_remove_solution(test_mp): # create a new instance of the transport problem and solve it mp = test_mp - scen = make_dantzig(mp, solve=test_data_path) + scen = make_dantzig(mp, solve=True) assert np.isclose(scen.var('z')['lvl'], 153.675) # check that re-solving the model will raise an error if a solution exists pytest.raises(ValueError, scen.solve, - model=str(test_data_path / 'transport_ixmp'), case='fail') + model='dantzig', case='fail') # remove the solution, check that variables are empty # and timeseries not marked `meta=True` are removed @@ -96,10 +96,10 @@ def get_distance(scen): ) -def test_multi_db_run(tmpdir, test_data_path): +def test_multi_db_run(tmpdir): # create a new instance of the transport problem and solve it mp1 = ixmp.Platform(backend='jdbc', driver='hsqldb', path=tmpdir / 'mp1') - scen1 = make_dantzig(mp1, solve=test_data_path) + scen1 = make_dantzig(mp1, solve=True) mp2 = ixmp.Platform(backend='jdbc', driver='hsqldb', path=tmpdir / 'mp2') # add other unit to make sure that the mapping is correct during clone diff --git a/tests/test_reporting.py b/tests/test_reporting.py index c21131953..d301b319a 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -135,8 +135,8 @@ def test_reporter_from_scenario(scenario): assert 'scenario' in r.graph -def test_reporter_from_dantzig(test_mp, test_data_path): - scen = make_dantzig(test_mp, solve=test_data_path) +def test_reporter_from_dantzig(test_mp): + scen = make_dantzig(test_mp, solve=True) # Reporter.from_scenario can handle the Dantzig problem rep = Reporter.from_scenario(scen) From c9f2ffe99e9c3001a8dbe809fb3a4a8d0bc5e96e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 12:56:17 +0100 Subject: [PATCH 08/21] Update RELEASE_NOTES --- RELEASE_NOTES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index fca0ac4bd..3acc8e9d7 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -4,6 +4,7 @@ Next release All changes ----------- +- `#212 `_: Add :meth:`Model.initialize` API to help populate new Scenarios according to a model scheme. - `#261 `_: Increase minimum pandas version to 1.0; adjust for `API changes and deprecations `_. - `#243 `_: Add :meth:`.export_timeseries_data` to write data for multiple scenarios to CSV. From 945133d54c0e797b5e78de15a0d6fbd305f7ecfe Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Thu, 21 Nov 2019 14:25:16 +0100 Subject: [PATCH 09/21] Add test placeholders --- tests/test_model.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_model.py b/tests/test_model.py index ea6f415df..5e7b61c95 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -10,3 +10,15 @@ def test_GAMSModel(test_mp, test_data_path, kwargs): s = make_dantzig(test_mp) s.solve(model='dantzig', **kwargs) + + +def test_model_initialize(): + # TODO Model.initialize() runs on both a 'empty' and already-init'd + # Scenario + + # TODO Unrecognized Scenario(scheme=...) raises an intelligible exception + + # TODO Keyword arguments to Scenario(...) that are not recognized by + # Model.initialize() raise an intelligible exception + + pass From 2edfff3b5a8cf56483ba6fdf0449263b90ea687b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 13:05:46 +0100 Subject: [PATCH 10/21] Adjust usage of testing.make_dantzig in R tests --- rixmp/tests/testthat/test_core.R | 6 +----- rixmp/tests/testthat/test_reporting.R | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/rixmp/tests/testthat/test_core.R b/rixmp/tests/testthat/test_core.R index 3ab1b2f83..706496708 100644 --- a/rixmp/tests/testthat/test_core.R +++ b/rixmp/tests/testthat/test_core.R @@ -60,11 +60,7 @@ test_that('set, mapping sets and par values can be set on a Scenario', { test_that('the canning problem can be solved', { # Create the Scenario mp <- test_mp() - scen <- ixmp$testing$make_dantzig(mp) - - # Solve - model_path = file.path(Sys.getenv('IXMP_TEST_DATA_PATH'), 'transport_ixmp') - scen$solve(model = model_path) + scen <- ixmp$testing$make_dantzig(mp, solve = TRUE) # Check value expect_equal(scen$var('z')$lvl, 153.675, tolerance = 1e-5) diff --git a/rixmp/tests/testthat/test_reporting.R b/rixmp/tests/testthat/test_reporting.R index 97fac737e..d4f6638d7 100644 --- a/rixmp/tests/testthat/test_reporting.R +++ b/rixmp/tests/testthat/test_reporting.R @@ -1,11 +1,7 @@ test_that('the canning problem can be reported', { # Create the Scenario mp <- test_mp() - scen <- ixmp$testing$make_dantzig(mp) - - # Solve - model_path = file.path(Sys.getenv('IXMP_TEST_DATA_PATH'), 'transport_ixmp') - scen$solve(model = model_path) + scen <- ixmp$testing$make_dantzig(mp, solve = TRUE) # Reporter.from_scenario can handle the Dantzig problem rep <- ixmp$reporting$Reporter$from_scenario(scen) From 34f78deed7cf1e99eda95de5f65300ca735a6090 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 13:42:18 +0100 Subject: [PATCH 11/21] Raise RuntimeError from Backend.commit() --- ixmp/backend/base.py | 5 +++++ ixmp/backend/jdbc.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index 5db652f7a..4092f62b5 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -296,6 +296,11 @@ def init_ts(self, ts: TimeSeries, annotation=None): Returns ------- None + + Raises + ------ + RuntimeError + if *ts* is newly created and :meth:`commit` has not been called. """ @abstractmethod diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index ee4a0f28d..370ce2153 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -445,7 +445,10 @@ def get(self, ts, version): ts.scheme = jobj.getScheme() def check_out(self, ts, timeseries_only): - self.jindex[ts].checkOut(timeseries_only) + try: + self.jindex[ts].checkOut(timeseries_only) + except java.IxException as e: + raise RuntimeError(e) def commit(self, ts, comment): self.jindex[ts].commit(comment) From a4b98aaa6d0afb9dc99408cc776fc5387e2b8074 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 13:43:14 +0100 Subject: [PATCH 12/21] Model.initialize() works on already-initialized Scenario --- ixmp/core.py | 12 +++++------- ixmp/model/base.py | 17 ++++++++++++++++- tests/test_model.py | 30 +++++++++++++++++++++++++----- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/ixmp/core.py b/ixmp/core.py index 551171c6b..ce9664d96 100644 --- a/ixmp/core.py +++ b/ixmp/core.py @@ -617,6 +617,7 @@ def __init__(self, mp, model, scenario, version=None, scheme=None, # Set attributes self.platform = mp + self.scheme = scheme self.model = model self.scenario = scenario @@ -636,13 +637,10 @@ def _cache(self): return hasattr(self.platform._backend, '_cache') # Retrieve the Model class correlating to the *scheme* - try: - model_class = get_model(scheme).__class__ - except KeyError: - pass - else: - # Use the model class to initialize the Scenario - model_class.initialize(self, **model_init_args) + model_class = get_model(scheme).__class__ + + # Use the model class to initialize the Scenario + model_class.initialize(self, **model_init_args) @classmethod def from_url(cls, url, errors='warn'): diff --git a/ixmp/model/base.py b/ixmp/model/base.py index 00d9c2a1b..611f5e22e 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -1,6 +1,11 @@ +import logging + from abc import ABC, abstractmethod +log = logging.getLogger(__name__) + + class Model(ABC): # pragma: no cover #: Name of the model. name = 'base' @@ -31,7 +36,8 @@ def initialize(cls, scenario): -------- initialize_items """ - pass + log.debug('No initialization for {!r}-scheme Scenario' + .format(scenario.scheme)) @classmethod def initialize_items(cls, scenario, items): @@ -57,6 +63,15 @@ def initialize_items(cls, scenario, items): .init_set .init_var """ + try: + # If *scenario* is already committed to the Backend, it must be + # checked out. + scenario.check_out() + except RuntimeError: + # If *scenario* is new (has not been committed), the checkout + # attempt raises an exception + pass + for item_info in items: # Copy so that pop() below does not modify *items* item_info = item_info.copy() diff --git a/tests/test_model.py b/tests/test_model.py index 5e7b61c95..caa4d9ac0 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,11 @@ -from ixmp.testing import make_dantzig +import logging + import pytest +from ixmp import Scenario +from ixmp.testing import make_dantzig +from ixmp.model.dantzig import DantzigModel + @pytest.mark.parametrize('kwargs', [ dict(comment=None), @@ -12,11 +17,26 @@ def test_GAMSModel(test_mp, test_data_path, kwargs): s.solve(model='dantzig', **kwargs) -def test_model_initialize(): - # TODO Model.initialize() runs on both a 'empty' and already-init'd - # Scenario +def test_model_initialize(test_mp, caplog): + # Model.initialize runs on an empty Scenario + s = make_dantzig(test_mp) + b = s.par('b') + assert len(b) == 3 + + # TODO modify a value for 'b' and ensure it is not overwritten when + # initialize is called again. + + # Model.initialize runs on an already initialized Scenario + DantzigModel.initialize(s, with_data=True) + assert len(s.par('b')) == 3 - # TODO Unrecognized Scenario(scheme=...) raises an intelligible exception + # Unrecognized Scenario(scheme=...) is initialized using the base method, a + # no-op + caplog.set_level(logging.DEBUG) + Scenario(test_mp, model='model name', scenario='scenario name', + version='new', scheme='unknown') + assert caplog.records[-1].message == \ + "No initialization for 'unknown'-scheme Scenario" # TODO Keyword arguments to Scenario(...) that are not recognized by # Model.initialize() raise an intelligible exception From ea24d8f2f4ede4b2017cbc4832bebe41f556e222 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 13:45:44 +0100 Subject: [PATCH 13/21] Adjust level of caplog fixture in test_run_clone --- tests/test_integration.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index cb9ec91bc..be4a9f7e8 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,8 @@ +import logging + +import numpy as np from numpy.testing import assert_array_equal from pandas.testing import assert_frame_equal -import numpy as np import pytest import ixmp @@ -11,6 +13,8 @@ def test_run_clone(test_mp, caplog): + caplog.set_level(logging.WARNING) + # this test is designed to cover the full functionality of the GAMS API # - initialize a new platform instance # - creates a new scenario and exports a gdx file From 3e9a18e529acd2931e889edcfe26ad886105f00b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 13:54:08 +0100 Subject: [PATCH 14/21] Remove 'raise from' in JDBCBackend.item_set_elements - JPype exceptions do not support this syntax --- ixmp/backend/jdbc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index 370ce2153..82a16c07b 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -754,7 +754,7 @@ def item_set_elements(self, s, type, name, elements): # Re-raise as Python ValueError raise ValueError(msg) from e else: # pragma: no cover - raise RuntimeError('unhandled Java exception') from e + raise RuntimeError(str(e)) self.cache_invalidate(s, type, name) From 044489d1bca91b116e7738c9de221fd1ed9e42ba Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 14:25:29 +0100 Subject: [PATCH 15/21] Further document base.Model.initialize --- ixmp/model/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ixmp/model/base.py b/ixmp/model/base.py index 611f5e22e..7ff41bdd7 100644 --- a/ixmp/model/base.py +++ b/ixmp/model/base.py @@ -27,6 +27,13 @@ def __init__(self, name, **kwargs): def initialize(cls, scenario): """Set up *scenario* with required items. + Implementations of :meth:`initialize`: + + - **may** add sets, set elements, and/or parameter values. + - **may** accept any number of keyword arguments to control behaviour. + - **must not** modify existing parameter data in *scenario*, either by + deleting or overwriting values. + Parameters ---------- scenario : .Scenario From 25e3ac6cceee841e111fa021754575ec6d94ca91 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 14:25:44 +0100 Subject: [PATCH 16/21] Add utils.update_par --- doc/source/api-python.rst | 1 + ixmp/utils.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/doc/source/api-python.rst b/doc/source/api-python.rst index cf4d970cb..0ac84cf73 100644 --- a/doc/source/api-python.rst +++ b/doc/source/api-python.rst @@ -203,6 +203,7 @@ Utilities --------- .. currentmodule:: ixmp.utils + :members: update_par .. automethod:: ixmp.utils.parse_url diff --git a/ixmp/utils.py b/ixmp/utils.py index 04b090cfd..09439e2c2 100644 --- a/ixmp/utils.py +++ b/ixmp/utils.py @@ -290,3 +290,16 @@ def describe(df): ]) return lines + + +def update_par(scenario, name, data): + """Update parameter *name* in *scenario* using *data*, without overwriting. + + Only values which do not already appear in the parameter data are added. + """ + tmp = pd.concat([scenario.par(name), data]) + columns = list(filter(lambda c: c != 'value', tmp.columns)) + tmp = tmp.drop_duplicates(subset=columns, keep=False) + + if len(tmp): + scenario.add_par(name, tmp) From 90047e35c9b04fa0e207d20520c29e7634fddcaa Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 14:26:20 +0100 Subject: [PATCH 17/21] Complete test_model_initialize --- ixmp/model/dantzig.py | 51 ++++++++++++++++++++++++++----------------- tests/test_model.py | 35 ++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index 4f6f5ea1d..ffdaeb0c4 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -3,6 +3,7 @@ import pandas as pd +from ixmp.utils import update_par from .gams import GAMSModel @@ -27,6 +28,29 @@ dict(ix_type='equ', name='supply', idx_sets=['i']), ) +DATA = { + 'i': ['seattle', 'san-diego'], + 'j': ['new-york', 'chicago', 'topeka'], + 'a': pd.DataFrame([ + ['seattle', 350, 'cases'], + ['san-diego', 600, 'cases'], + ], columns='i value unit'.split()), + 'b': pd.DataFrame([ + ['new-york', 325, 'cases'], + ['chicago', 300, 'cases'], + ['topeka', 275, 'cases'], + ], columns='j value unit'.split()), + 'd': pd.DataFrame([ + ['seattle', 'new-york', 2.5, 'km'], + ['seattle', 'chicago', 1.7, 'km'], + ['seattle', 'topeka', 1.8, 'km'], + ['san-diego', 'new-york', 2.5, 'km'], + ['san-diego', 'chicago', 1.8, 'km'], + ['san-diego', 'topeka', 1.4, 'km'], + ], columns='i j value unit'.split()), + 'f': (90.0, 'USD_per_km'), +} + class DantzigModel(GAMSModel): """Dantzig's cannery/transport problem as a :class:`GAMSModel`. @@ -55,26 +79,13 @@ def initialize(cls, scenario, with_data=False): return # Add set elements - scenario.add_set('i', ['seattle', 'san-diego']) - scenario.add_set('j', ['new-york', 'chicago', 'topeka']) + scenario.add_set('i', DATA['i']) + scenario.add_set('j', DATA['j']) # Add parameter values - scenario.add_par('a', 'seattle', 350, 'cases') - scenario.add_par('a', 'san-diego', 600, 'cases') - - scenario.add_par('b', pd.DataFrame([ - ['new-york', 325, 'cases'], - ['chicago', 300, 'cases'], - ['topeka', 275, 'cases'], - ], columns='j value unit'.split())) - - scenario.add_par('d', pd.DataFrame([ - ['seattle', 'new-york', 2.5, 'km'], - ['seattle', 'chicago', 1.7, 'km'], - ['seattle', 'topeka', 1.8, 'km'], - ['san-diego', 'new-york', 2.5, 'km'], - ['san-diego', 'chicago', 1.8, 'km'], - ['san-diego', 'topeka', 1.4, 'km'], - ], columns='i j value unit'.split())) + update_par(scenario, 'a', DATA['a']) + update_par(scenario, 'b', DATA['b']) + update_par(scenario, 'd', DATA['d']) - scenario.change_scalar('f', 90.0, 'USD_per_km') + # TODO avoid overwriting the existing value + scenario.change_scalar('f', *DATA['f']) diff --git a/tests/test_model.py b/tests/test_model.py index caa4d9ac0..b75a7e276 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -20,15 +20,22 @@ def test_GAMSModel(test_mp, test_data_path, kwargs): def test_model_initialize(test_mp, caplog): # Model.initialize runs on an empty Scenario s = make_dantzig(test_mp) - b = s.par('b') - assert len(b) == 3 + b1 = s.par('b') + assert len(b1) == 3 - # TODO modify a value for 'b' and ensure it is not overwritten when - # initialize is called again. + # Modify a value for 'b' + s.check_out() + s.add_par('b', 'chicago', 600, 'cases') + s.commit('Overwrite b(chicago)') - # Model.initialize runs on an already initialized Scenario + # Model.initialize runs on an already-initialized Scenario, without error DantzigModel.initialize(s, with_data=True) - assert len(s.par('b')) == 3 + + # Data has the same length... + b2 = s.par('b') + assert len(b2) == 3 + # ...but modified value(s) are not overwritten + assert (b2.query("j == 'chicago'")['value'] == 600).all() # Unrecognized Scenario(scheme=...) is initialized using the base method, a # no-op @@ -38,7 +45,15 @@ def test_model_initialize(test_mp, caplog): assert caplog.records[-1].message == \ "No initialization for 'unknown'-scheme Scenario" - # TODO Keyword arguments to Scenario(...) that are not recognized by - # Model.initialize() raise an intelligible exception - - pass + # Keyword arguments to Scenario(...) that are not recognized by + # Model.initialize() raise an intelligible exception + with pytest.raises(TypeError, + match="unexpected keyword argument 'bad_arg1'"): + Scenario(test_mp, model='model name', scenario='scenario name', + version='new', scheme='unknown', bad_arg1=111) + + with pytest.raises(TypeError, + match="unexpected keyword argument 'bad_arg2'"): + Scenario(test_mp, model='model name', scenario='scenario name', + version='new', scheme='dantzig', with_data=True, + bad_arg2=222) From a5a12aedd2b620ef78e074013b3f085e81e556bb Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 17 Dec 2019 14:31:29 +0100 Subject: [PATCH 18/21] Correct git merge of Scenario.__init__ --- ixmp/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ixmp/core.py b/ixmp/core.py index ce9664d96..eef2e4ff5 100644 --- a/ixmp/core.py +++ b/ixmp/core.py @@ -632,16 +632,16 @@ def __init__(self, mp, model, scenario, version=None, scheme=None, warn('Using `ixmp.Scenario` for MESSAGE-scheme scenarios is ' 'deprecated, please use `message_ix.Scenario`') - @property - def _cache(self): - return hasattr(self.platform._backend, '_cache') - # Retrieve the Model class correlating to the *scheme* model_class = get_model(scheme).__class__ # Use the model class to initialize the Scenario model_class.initialize(self, **model_init_args) + @property + def _cache(self): + return hasattr(self.platform._backend, '_cache') + @classmethod def from_url(cls, url, errors='warn'): """Instantiate a Scenario given an ixmp-scheme URL. From 04c59c07de32ae54e12b44fbac05c4cec82848f8 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 10 Jan 2020 00:54:37 +0100 Subject: [PATCH 19/21] Use a temporary directory for GAMSModel input/output GDX files --- ixmp/model/gams.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index 027e8edce..d1d1343ec 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -1,6 +1,7 @@ import os from pathlib import Path from subprocess import check_call +from tempfile import TemporaryDirectory from ixmp.backend import ItemType @@ -77,15 +78,17 @@ class GAMSModel(Model): defaults = { 'model_file': '{model_name}.gms', 'case': "{scenario.model}_{scenario.scenario}", - 'in_file': '{model_name}_in.gdx', - 'out_file': '{model_name}_out.gdx', + 'in_file': str(Path('{temp_dir}', '{model_name}_in.gdx')), + 'out_file': str(Path('{temp_dir}', '{model_name}_out.gdx')), 'solve_args': ['--in="{in_file}"', '--out="{out_file}"'], + # Not formatted 'gams_args': ['LogOption=4'], 'check_solution': True, 'comment': None, 'equ_list': None, 'var_list': None, + 'use_temp_dir': True, } def __init__(self, name=None, **model_options): @@ -99,6 +102,12 @@ def run(self, scenario): self.scenario = scenario + if self.use_temp_dir: + # Create a temporary directory; automatically deleted at the end of + # the context + _temp_dir = TemporaryDirectory() + self.temp_dir = _temp_dir.name + def format(key): value = getattr(self, key) try: From 7997fa13a842d15b777856a3f7decf00667965a1 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 7 Feb 2020 21:44:36 +0100 Subject: [PATCH 20/21] Debug Windows tests 1 --- ixmp/model/gams.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ixmp/model/gams.py b/ixmp/model/gams.py index d1d1343ec..876b13221 100644 --- a/ixmp/model/gams.py +++ b/ixmp/model/gams.py @@ -144,7 +144,8 @@ def format(key): 'write to GDX files, e.g. JDBCBackend') # Invoke GAMS - check_call(command, shell=os.name == 'nt', cwd=model_file.parent) + cwd = self.temp_dir if self.use_temp_dir else model_file.parent + check_call(command, shell=os.name == 'nt', cwd=cwd) # Read model solution backend.read_file(self.out_file, ItemType.MODEL, **s_arg, From e7be637ff20428e2852a612605836df6212ab802 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Fri, 7 Feb 2020 23:42:07 +0100 Subject: [PATCH 21/21] Correct tutorial references to model --- tutorial/transport/R_transport.ipynb | 2 +- tutorial/transport/R_transport_scenario.ipynb | 2 +- tutorial/transport/py_transport.ipynb | 2 +- tutorial/transport/py_transport_scenario.ipynb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tutorial/transport/R_transport.ipynb b/tutorial/transport/R_transport.ipynb index 281a1d480..c65a81d82 100644 --- a/tutorial/transport/R_transport.ipynb +++ b/tutorial/transport/R_transport.ipynb @@ -322,7 +322,7 @@ }, "outputs": [], "source": [ - "scen$solve(model=\"transport_ixmp\")" + "scen$solve(model='dantzig')" ] }, { diff --git a/tutorial/transport/R_transport_scenario.ipynb b/tutorial/transport/R_transport_scenario.ipynb index 24444a8be..729a32cda 100644 --- a/tutorial/transport/R_transport_scenario.ipynb +++ b/tutorial/transport/R_transport_scenario.ipynb @@ -230,7 +230,7 @@ }, "outputs": [], "source": [ - "scen_detroit$solve(model='transport_ixmp')" + "scen_detroit$solve(model='dantzig')" ] }, { diff --git a/tutorial/transport/py_transport.ipynb b/tutorial/transport/py_transport.ipynb index ef92a6bde..efac6e40d 100644 --- a/tutorial/transport/py_transport.ipynb +++ b/tutorial/transport/py_transport.ipynb @@ -321,7 +321,7 @@ "metadata": {}, "outputs": [], "source": [ - "scen.solve(model='transport_ixmp')" + "scen.solve(model='dantzig')" ] }, { diff --git a/tutorial/transport/py_transport_scenario.ipynb b/tutorial/transport/py_transport_scenario.ipynb index 40fd781ec..f32bd7dfc 100644 --- a/tutorial/transport/py_transport_scenario.ipynb +++ b/tutorial/transport/py_transport_scenario.ipynb @@ -201,7 +201,7 @@ "metadata": {}, "outputs": [], "source": [ - "scen_detroit.solve(model='transport_ixmp')" + "scen_detroit.solve(model='dantzig')" ] }, {