From eca6e0647656728c023efd3943cf113791910a3b Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 12 May 2020 23:57:48 +0200 Subject: [PATCH 1/6] Add model.pyomo --- ixmp/model/pyomo.py | 134 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 ixmp/model/pyomo.py diff --git a/ixmp/model/pyomo.py b/ixmp/model/pyomo.py new file mode 100644 index 000000000..39adaab05 --- /dev/null +++ b/ixmp/model/pyomo.py @@ -0,0 +1,134 @@ +from inspect import signature + +try: + from pyomo import environ as pyo, opt + has_pyomo = True +except ImportError: + has_pyomo = False + +from ixmp.model.base import Model + + +COMPONENT = dict( + par=pyo.Param, + set=pyo.Set, + var=pyo.Var, + equ=pyo.Constraint, +) + + +def get_sets(model, names): + return [model.component(idx_set) for idx_set in names] + + +class PyomoModel(Model): + """General class for ixmp models using :mod:`pyomo`.""" + name = 'pyomo' + + items = {} + constraints = {} + objective = None + _partials = {} + + def __init__(self, name=None, solver='glpk'): + if not has_pyomo: + raise ImportError('pyomo must be installed') + + self.opt = opt.SolverFactory(solver) + + m = pyo.AbstractModel() + + for name, info in self.items.items(): + if name == self.objective: + # Handle the objective separately + continue + + Component = COMPONENT[info['ix_type']] + + kwargs = {} + + if info['ix_type'] == 'equ': + func = self.equation[name] + params = signature(func).parameters + idx_sets = list(params.keys())[1:] + kwargs = dict(rule=func) + else: + idx_sets = info.get('idx_sets', None) or [] + + # NB would like to do this, but pyomo doesn't recognize partial + # objects as callable + # if info['ix_type'] != 'var': + # kwarg = dict( + # initialize=partial(self.to_pyomo, name) + # ) + + kwargs.update(self.component_kwargs.get(name, {})) + + component = Component(*get_sets(m, idx_sets), **kwargs) + m.add_component(name, component) + + obj_func = self.equation[self.objective] + obj = pyo.Objective(rule=obj_func, sense=pyo.minimize) + m.add_component(self.objective, obj) + + # Store + self.model = m + + def to_pyomo(self, name): + info = self.items[name] + ix_type = info['ix_type'] + + if ix_type == 'par': + item = self.scenario.par(name) + + idx_sets = info.get('idx_sets', []) or [] + if len(idx_sets): + series = item.set_index(idx_sets)['value'] + series.index = series.index.to_flat_index() + return series.to_dict() + else: + return {None: item['value']} + elif ix_type == 'set': + return {None: self.scenario.set(name).tolist()} + + def all_to_pyomo(self): + return {None: dict( + filter( + lambda name_data: name_data[1], + [(name, self.to_pyomo(name)) for name in self.items] + ) + )} + + def all_from_pyomo(self, model): + for name, info in self.items.items(): + if info['ix_type'] not in ('equ', 'var'): + continue + self.from_pyomo(model, name) + + def from_pyomo(self, model, name): + component = model.component(name) + component.display() + try: + data = component.get_values() + except Exception as exc: + print(exc) + return + + # TODO add to Scenario; currently not possible because ixmp_source does + # not allow setting elements of 'equ' and 'var' + del data + + def run(self, scenario): + self.scenario = scenario + + data = self.all_to_pyomo() + + m = self.model.create_instance(data=data) + + assert m.is_constructed() + + results = self.opt.solve(m) + + self.all_from_pyomo(m) + + delattr(self, 'scenario') From 75975b1587c98adb28777ca6065fbd94f8a1333d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 12 May 2020 23:58:37 +0200 Subject: [PATCH 2/6] Split DantzigModel to GAMS and Pyomo versions --- ixmp/__init__.py | 6 ++-- ixmp/model/dantzig.py | 72 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index f207cfa8f..819b57412 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -9,7 +9,7 @@ from ixmp.core.scenario import Scenario, TimeSeries from ixmp.model import MODELS from ixmp.model.base import ModelError -from ixmp.model.dantzig import DantzigModel +from ixmp.model.dantzig import DantzigGAMSModel, DantzigPyomoModel from ixmp.model.gams import GAMSModel from ixmp.reporting import Reporter from ixmp.utils import show_versions @@ -41,7 +41,9 @@ { "default": GAMSModel, "gams": GAMSModel, - "dantzig": DantzigModel, + "dantzig": DantzigGAMSModel, + "dantzig-gams": DantzigGAMSModel, + "dantzig-pyomo": DantzigPyomoModel, } ) diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index 2392c0018..000cb51a6 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -1,11 +1,13 @@ from collections import ChainMap +from functools import lru_cache from pathlib import Path import pandas as pd from ixmp.utils import maybe_check_out, maybe_commit, update_par - from .gams import GAMSModel +from .pyomo import PyomoModel + ITEMS = { # Plants @@ -61,22 +63,9 @@ } -class DantzigModel(GAMSModel): - """Dantzig's cannery/transport problem as a :class:`GAMSModel`. - - Provided for testing :mod:`ixmp` code. - """ - +class DantzigModel: name = "dantzig" - defaults = ChainMap( - { - # Override keys from GAMSModel - "model_file": Path(__file__).with_name("dantzig.gms"), - }, - GAMSModel.defaults, - ) - @classmethod def initialize(cls, scenario, with_data=False): """Initialize the problem. @@ -106,3 +95,56 @@ def initialize(cls, scenario, with_data=False): scenario.change_scalar("f", *DATA["f"]) maybe_commit(scenario, checkout, f"{cls.__name__}.initialize") + + +class DantzigGAMSModel(DantzigModel, GAMSModel): + """Dantzig's cannery/transport problem as a :class:`GAMSModel`. + + Provided for testing :mod:`ixmp` code. + """ + + name = "dantzig-gams" + + defaults = ChainMap( + { + # Override keys from GAMSModel + "model_file": Path(__file__).with_name("dantzig.gms"), + }, + GAMSModel.defaults, + ) + + +def supply(model, i): + return sum(model.x[i, j] for j in model.j) <= model.a[i] + + +def demand(model, j): + return sum(model.x[i, j] for i in model.i) >= model.b[j] + + +@lru_cache() +def c(model, i, j): + return model.f * model.d[i, j] / 1000 + + +def cost(model): + return sum(c(model, i, j) * model.x[i, j] for i in model.i for j in model.j) + + +class DantzigPyomoModel(DantzigModel, PyomoModel): + """Dantzig's cannery/transport problem as a :class:`PyomoModel`. + + Provided for testing :mod:`ixmp` code. + """ + + name = "dantzig-pyomo" + items = ITEMS + equation = dict( + supply=supply, + demand=demand, + cost=cost, + ) + component_kwargs = dict( + x=dict(bounds=(0.0, None)), + ) + objective = "cost" From 514ce0876823ddd645f1d2cac913ce10c0be34ec Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Wed, 13 May 2020 00:01:13 +0200 Subject: [PATCH 3/6] Add test of DantzigPyomoModel --- ixmp/testing/data.py | 6 ++++-- ixmp/tests/test_model.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ixmp/testing/data.py b/ixmp/testing/data.py index 45519a373..6989af5eb 100644 --- a/ixmp/testing/data.py +++ b/ixmp/testing/data.py @@ -155,7 +155,9 @@ def add_test_data(scen: Scenario): return t, t_foo, t_bar, x -def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scenario: +def make_dantzig( + mp: Platform, solve: bool = False, quiet: bool = False, scheme="dantzig-gams" +) -> Scenario: """Return :class:`ixmp.Scenario` of Dantzig's canning/transport problem. Parameters @@ -191,7 +193,7 @@ def make_dantzig(mp: Platform, solve: bool = False, quiet: bool = False) -> Scen **models["dantzig"], # type: ignore [arg-type] version="new", annotation=annot, - scheme="dantzig", + scheme=scheme, with_data=True, ) diff --git a/ixmp/tests/test_model.py b/ixmp/tests/test_model.py index 8187decff..e0783e02d 100644 --- a/ixmp/tests/test_model.py +++ b/ixmp/tests/test_model.py @@ -21,6 +21,26 @@ class M1(Model): M1() +@pytest.mark.parametrize( + "kwargs", + [ + dict(comment=None), + dict(equ_list=None, var_list=["x"]), + dict(equ_list=["demand", "supply"], var_list=[]), + ], + ids=["null-comment", "null-list", "empty-list"], +) +def test_GAMSModel(test_mp, test_data_path, kwargs): + s = make_dantzig(test_mp) + s.solve(model="dantzig", **kwargs) + + +def test_PyomoModel(test_mp): + """Pyomo version of the Dantzig model builds and solves.""" + s = make_dantzig(test_mp, scheme="dantzig-pyomo") + s.solve(model="dantzig-pyomo") + + def test_model_initialize(test_mp, caplog): # Model.initialize runs on an empty Scenario s = make_dantzig(test_mp) From edd1568e813b4909a54ac8b1bdb774aa60dafe42 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 29 Jun 2021 17:58:18 +0200 Subject: [PATCH 4/6] Blacked .model.pyomo; add type hints --- ixmp/model/pyomo.py | 55 ++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/ixmp/model/pyomo.py b/ixmp/model/pyomo.py index 39adaab05..46aa5a4d2 100644 --- a/ixmp/model/pyomo.py +++ b/ixmp/model/pyomo.py @@ -1,14 +1,16 @@ from inspect import signature +from typing import Dict, Optional try: - from pyomo import environ as pyo, opt + from pyomo import environ as pyo + from pyomo import opt + has_pyomo = True except ImportError: has_pyomo = False from ixmp.model.base import Model - COMPONENT = dict( par=pyo.Param, set=pyo.Set, @@ -23,16 +25,17 @@ def get_sets(model, names): class PyomoModel(Model): """General class for ixmp models using :mod:`pyomo`.""" - name = 'pyomo' - items = {} - constraints = {} - objective = None - _partials = {} + name = "pyomo" + + items: Dict[str, dict] = {} + constraints: Dict = {} + objective: Optional[str] = None + _partials: Dict = {} - def __init__(self, name=None, solver='glpk'): + def __init__(self, name=None, solver="glpk"): if not has_pyomo: - raise ImportError('pyomo must be installed') + raise ImportError("pyomo must be installed") self.opt = opt.SolverFactory(solver) @@ -43,17 +46,17 @@ def __init__(self, name=None, solver='glpk'): # Handle the objective separately continue - Component = COMPONENT[info['ix_type']] + Component = COMPONENT[info["ix_type"]] kwargs = {} - if info['ix_type'] == 'equ': + if info["ix_type"] == "equ": func = self.equation[name] params = signature(func).parameters idx_sets = list(params.keys())[1:] kwargs = dict(rule=func) else: - idx_sets = info.get('idx_sets', None) or [] + idx_sets = info.get("idx_sets", None) or [] # NB would like to do this, but pyomo doesn't recognize partial # objects as callable @@ -76,32 +79,34 @@ def __init__(self, name=None, solver='glpk'): def to_pyomo(self, name): info = self.items[name] - ix_type = info['ix_type'] + ix_type = info["ix_type"] - if ix_type == 'par': + if ix_type == "par": item = self.scenario.par(name) - idx_sets = info.get('idx_sets', []) or [] + idx_sets = info.get("idx_sets", []) or [] if len(idx_sets): - series = item.set_index(idx_sets)['value'] + series = item.set_index(idx_sets)["value"] series.index = series.index.to_flat_index() return series.to_dict() else: - return {None: item['value']} - elif ix_type == 'set': + return {None: item["value"]} + elif ix_type == "set": return {None: self.scenario.set(name).tolist()} def all_to_pyomo(self): - return {None: dict( - filter( - lambda name_data: name_data[1], - [(name, self.to_pyomo(name)) for name in self.items] + return { + None: dict( + filter( + lambda name_data: name_data[1], + [(name, self.to_pyomo(name)) for name in self.items], + ) ) - )} + } def all_from_pyomo(self, model): for name, info in self.items.items(): - if info['ix_type'] not in ('equ', 'var'): + if info["ix_type"] not in ("equ", "var"): continue self.from_pyomo(model, name) @@ -131,4 +136,4 @@ def run(self, scenario): self.all_from_pyomo(m) - delattr(self, 'scenario') + delattr(self, "scenario") From 2a45f0904cce1dca4e46e90b42ff227b18b573cf Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 29 Jun 2021 17:59:18 +0200 Subject: [PATCH 5/6] Satisfy mypy in __init___, .model.dantzig --- ixmp/__init__.py | 18 ++++++++---------- ixmp/model/dantzig.py | 5 +++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ixmp/__init__.py b/ixmp/__init__.py index 819b57412..215ebf39f 100644 --- a/ixmp/__init__.py +++ b/ixmp/__init__.py @@ -37,16 +37,14 @@ BACKENDS["jdbc"] = JDBCBackend # Register Models provided by ixmp -MODELS.update( - { - "default": GAMSModel, - "gams": GAMSModel, - "dantzig": DantzigGAMSModel, - "dantzig-gams": DantzigGAMSModel, - "dantzig-pyomo": DantzigPyomoModel, - } -) - +for name, cls in ( + ("default", GAMSModel), + ("gams", GAMSModel), + ("dantzig", DantzigGAMSModel), + ("dantzig-gams", DantzigGAMSModel), + ("dantzig-pyomo", DantzigPyomoModel), +): + MODELS[name] = cls # Configure the 'ixmp' logger: write messages to stdout, defaulting to level WARNING # and above diff --git a/ixmp/model/dantzig.py b/ixmp/model/dantzig.py index 000cb51a6..7f9f3205b 100644 --- a/ixmp/model/dantzig.py +++ b/ixmp/model/dantzig.py @@ -1,15 +1,16 @@ from collections import ChainMap from functools import lru_cache from pathlib import Path +from typing import Dict import pandas as pd from ixmp.utils import maybe_check_out, maybe_commit, update_par + from .gams import GAMSModel from .pyomo import PyomoModel - -ITEMS = { +ITEMS: Dict[str, dict] = { # Plants "i": dict(ix_type="set"), # Markets From b36c34f52d0482d1da1aed668bfbbc3b0edb2a1e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 29 Jun 2021 17:59:38 +0200 Subject: [PATCH 6/6] Add pyomo to setup.cfg, pyproject.toml --- pyproject.toml | 1 + setup.cfg | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e9c73df96..eb97ed6a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,7 @@ module = [ "memory_profiler", "pandas.*", "pyam", + "pyomo", "pretenders.*", ] ignore_missing_imports = true diff --git a/setup.cfg b/setup.cfg index 7e1ffc46a..91dc7f0d5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,12 +47,15 @@ docs = sphinx >= 3.0 sphinx_rtd_theme sphinxcontrib-bibtex +pyomo = + pyomo report = genno[compat,graphviz] tutorial = jupyter tests = %(docs)s + %(pyomo)s %(report)s %(tutorial)s codecov