From 164513e9a0f91e694738d6d1398ac20e43867ea8 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Wed, 19 Feb 2020 18:15:48 +0000 Subject: [PATCH 01/19] :hammer: Add branching priorities to CPLEXSHELL - Include `branch_priority` and `branch_direction` attributes of variable data - Read off these attributes as part of the CPLEXSHELL solve and write them to a CPLEX .ord file - Update the CPLEX OPL script to read these .ord file should it exist --- pyomo/core/base/var.py | 10 ++- pyomo/core/tests/unit/test_var.py | 18 ++++ pyomo/solvers/plugins/solvers/CPLEX.py | 87 +++++++++++++++++++ pyomo/solvers/tests/checks/test_cplex.py | 102 ++++++++++++++++++++++- 4 files changed, 215 insertions(+), 2 deletions(-) diff --git a/pyomo/core/base/var.py b/pyomo/core/base/var.py index 82e5c788c20..b372d7c1183 100644 --- a/pyomo/core/base/var.py +++ b/pyomo/core/base/var.py @@ -11,6 +11,7 @@ __all__ = ['Var', '_VarData', '_GeneralVarData', 'VarList', 'SimpleVar'] import logging +from typing import Optional from weakref import ref as weakref_ref from pyomo.common.modeling import NoArgumentGiven @@ -278,6 +279,10 @@ def to_string(self, verbose=None, labeler=None, smap=None, compute_values=False) return self.name +PriorityType = int +BranchDirectionType = int + + class _GeneralVarData(_VarData): """ This class defines the data for a single variable. @@ -308,7 +313,7 @@ class _GeneralVarData(_VarData): these attributes in certain cases. """ - __slots__ = ('_value', '_lb', '_ub', '_domain', 'fixed', 'stale') + __slots__ = ('_value', '_lb', '_ub', '_domain', 'fixed', 'stale', 'branch_priority', 'branch_direction') def __init__(self, domain=Reals, component=None): # @@ -341,6 +346,9 @@ def __init__(self, domain=Reals, component=None): "for bounds (like a Pyomo Set). Examples: NonNegativeReals, " "Integers, Binary" % (domain, (RealSet, IntegerSet, BooleanSet))) + self.branch_priority = None # type: Optional[PriorityType] + self.branch_direction = None # type: Optional[BranchDirectionType] + def __getstate__(self): state = super(_GeneralVarData, self).__getstate__() for i in _GeneralVarData.__slots__: diff --git a/pyomo/core/tests/unit/test_var.py b/pyomo/core/tests/unit/test_var.py index 5faf0c44ef8..fe814428867 100644 --- a/pyomo/core/tests/unit/test_var.py +++ b/pyomo/core/tests/unit/test_var.py @@ -440,6 +440,24 @@ def test_value(self): self.assertEqual( type(tmp), int) self.assertEqual( tmp, 3 ) + def test_branch_priority_attr(self): + """Test branch_priority attribute""" + self.model.x = Var() + self.instance = self.model.create_instance() + + self.assertIsNone(self.instance.x.branch_priority) + self.instance.x.branch_priority = 1 + self.assertEqual(self.instance.x.branch_priority, 1) + + def test_branch_direction_attr(self): + """Test branch_direction attribute""" + self.model.x = Var() + self.instance = self.model.create_instance() + + self.assertIsNone(self.instance.x.branch_direction) + self.instance.x.branch_direction = -1 + self.assertEqual(self.instance.x.branch_direction, -1) + class TestArrayVar(TestSimpleVar): diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 25ee5651a36..015e0938449 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -112,6 +112,33 @@ def __new__(cls, *args, **kwds): return opt +class CPLEXBranchDirection: + default = 0 + down = -1 + up = 1 + + ALL = {default, down, up} + + @staticmethod + def to_str(branch_direction): + try: + return { + CPLEXBranchDirection.down: "DN", + CPLEXBranchDirection.up: "UP", + }[branch_direction] + except KeyError: + return "" + + +class ORDFileSchema: + HEADER = "* ENCODING=ISO-8859-1\nNAME Priority Order\n" + FOOTER = "ENDATA\n" + + @classmethod + def ROW(cls, name, priority, branch_direction=None): + return " %s %s %s\n" % (CPLEXBranchDirection.to_str(branch_direction), name, priority) + + @SolverFactory.register('_cplex_shell', doc='Shell interface to the CPLEX LP/MIP solver') class CPLEXSHELL(ILMLicensedSystemCallSolver): """Shell interface to the CPLEX LP/MIP solver @@ -201,6 +228,35 @@ def _warm_start(self, instance): mst_file.write("\n") mst_file.write("\n") + def _write_priorities_file(self, instance) -> None: + """ Write a variable priorities file in the CPLEX ORD format. """ + from pyomo.core.base import Var + + if isinstance(instance, IBlock): + smap = getattr(instance, "._symbol_maps")[self._smap_id] + else: + smap = instance.solutions.symbol_map[self._smap_id] + byObject = smap.byObject + + with open(self._priorities_file_name, "w") as ord_file: + ord_file.write(ORDFileSchema.HEADER) + for var in instance.component_data_objects(Var): + priority = var.branch_priority + if priority is None: + continue + + if not (0 <= priority == int(priority)): + raise ValueError("`priority` must be a non-negative integer") + + if id(var) not in byObject or not var.active: + continue + + ord_file.write( + ORDFileSchema.ROW(byObject[id(var)], priority, var.branch_direction) + ) + + ord_file.write(ORDFileSchema.FOOTER) + # over-ride presolve to extract the warm-start keyword, if specified. def _presolve(self, *args, **kwds): @@ -234,6 +290,21 @@ def _presolve(self, *args, **kwds): self._warm_start_file_name = pyutilib.services.TempfileManager.\ create_tempfile(suffix = '.cplex.mst') + self._priorities_solve = kwds.pop("priorities", False) + self._priorities_file_name = _validate_file_name( + self, kwds.pop("priorities_file", None), "branching priorities" + ) + user_priorities = self._priorities_file_name is not None + + if ( + self._priorities_solve + and not isinstance(args[0], basestring) + and not user_priorities + ): + self._priorities_file_name = pyutilib.services.TempfileManager.create_tempfile( + suffix=".cplex.ord" + ) + # let the base class handle any remaining keywords/actions. ILMLicensedSystemCallSolver._presolve(self, *args, **kwds) @@ -259,6 +330,16 @@ def _presolve(self, *args, **kwds): print("Warm start write time= %.2f seconds" % (end_time-start_time)) + if self._priorities_solve and (not user_priorities): + start_time = time.time() + self._write_priorities_file(args[0]) + end_time = time.time() + if self._report_timing: + print( + "Branching priorities write time= %.2f seconds" + % (end_time - start_time) + ) + def _default_executable(self): executable = pyomo.common.Executable("cplex") if not executable: @@ -328,6 +409,9 @@ def create_command_line(self, executable, problem_files): (self._warm_start_file_name is not None): script += 'read %s\n' % (self._warm_start_file_name,) + if self._priorities_solve and self._priorities_file_name is not None: + script += "read %s\n" % (self._priorities_file_name,) + if 'relax_integrality' in self.options: script += 'change problem lp\n' @@ -351,6 +435,9 @@ def create_command_line(self, executable, problem_files): print("Solver warm-start file=" +self._warm_start_file_name) + if self._priorities_solve and self._priorities_file_name is not None: + print("Solver priorities file=" + self._priorities_file_name) + # # Define command line # diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 3712a0feff8..480167cb06c 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -9,9 +9,15 @@ # ___________________________________________________________________________ import os + +import pyutilib import pyutilib.th as unittest -from pyomo.solvers.plugins.solvers.CPLEX import _validate_file_name +from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var +from pyomo.opt import ProblemFormat, convert_problem +from pyomo.solvers.plugins.solvers.CPLEX import (CPLEXSHELL, MockCPLEX, + _validate_file_name) + class _mock_cplex_128(object): def version(self): @@ -54,5 +60,99 @@ def test_validate_file_name(self): with self.assertRaisesRegexp(ValueError, msg): _validate_file_name(_128, fname, 'xxx') + +class CPLEXShellPrioritiesFile(unittest.TestCase): + def setUp(self): + from pyomo.solvers.plugins.converter.model import PyomoMIPConverter # register the `ProblemConverterFactory` + from pyomo.repn.plugins.cpxlp import ProblemWriter_cpxlp # register the `WriterFactory` + + self.mock_model = self.get_mock_model() + self.mock_cplex_shell = self.get_mock_cplex_shell(self.mock_model) + self.mock_cplex_shell._priorities_file_name = pyutilib.services.TempfileManager.create_tempfile( + suffix=".cplex.ord" + ) + + def tearDown(self): + pyutilib.services.TempfileManager.clear_tempfiles() + + def get_mock_model(self): + model = ConcreteModel() + model.x = Var(within=Binary) + model.con = Constraint(expr=model.x >= 1) + model.obj = Objective(expr=model.x) + return model + + def get_mock_cplex_shell(self, mock_model): + solver = MockCPLEX() + solver._problem_files, solver._problem_format, solver._smap_id = convert_problem( + (mock_model,), + ProblemFormat.cpxlp, + [ProblemFormat.cpxlp], + has_capability=lambda x: True, + ) + return solver + + def get_priorities_file_as_string(self, mock_cplex_shell): + with open(mock_cplex_shell._priorities_file_name, "r") as ord_file: + priorities_file = ord_file.read() + return priorities_file + + def test_write_empty_priorities_file(self): + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) + + with open(self.mock_cplex_shell._priorities_file_name, "r") as ord_file: + priorities_file = ord_file.read() + + self.assertEqual( + priorities_file, + "* ENCODING=ISO-8859-1\nNAME Priority Order\nENDATA\n", + ) + + def test_write_priority_to_priorities_file(self): + self.mock_model.x.branch_priority = 10 + + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) + priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) + + self.assertEqual( + priorities_file, + "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n", + ) + self.assertIsNone(self.mock_model.x.branch_direction) + + def test_write_priority_and_direction_to_priorities_file(self): + self.mock_model.x.branch_priority = 10 + self.mock_model.x.branch_direction = -1 + + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) + priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) + + self.assertEqual( + priorities_file, + "* ENCODING=ISO-8859-1\nNAME Priority Order\n DN x1 10\nENDATA\n", + ) + + def test_raise_due_to_invalid_priority(self): + self.mock_model.x.branch_priority = -1 + with self.assertRaises(ValueError): + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) + + self.mock_model.x.branch_priority = 1.1 + with self.assertRaises(ValueError): + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) + + def test_use_default_due_to_invalid_direction(self): + self.mock_model.x.branch_priority = 10 + self.mock_model.x.branch_direction = "invalid_branching_direction" + + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) + priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) + + self.assertEqual( + priorities_file, + "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n", + ) + + if __name__ == "__main__": unittest.main() From 81f0a6e6b2c23a188e069f772d051eaec83e8aa3 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Thu, 20 Feb 2020 14:18:09 +0000 Subject: [PATCH 02/19] :rotating_light: Add tests of solving with priorities - End-to-end tests of `opt.solve()` using branching priorities with CPLEXSHELL --- pyomo/solvers/tests/checks/test_cplex.py | 100 ++++++++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 480167cb06c..f3e5106072c 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -9,12 +9,13 @@ # ___________________________________________________________________________ import os +import sys import pyutilib import pyutilib.th as unittest -from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var -from pyomo.opt import ProblemFormat, convert_problem +from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum +from pyomo.opt import ProblemFormat, convert_problem, SolverFactory from pyomo.solvers.plugins.solvers.CPLEX import (CPLEXSHELL, MockCPLEX, _validate_file_name) @@ -61,7 +62,7 @@ def test_validate_file_name(self): _validate_file_name(_128, fname, 'xxx') -class CPLEXShellPrioritiesFile(unittest.TestCase): +class CPLEXShellWritePrioritiesFile(unittest.TestCase): def setUp(self): from pyomo.solvers.plugins.converter.model import PyomoMIPConverter # register the `ProblemConverterFactory` from pyomo.repn.plugins.cpxlp import ProblemWriter_cpxlp # register the `WriterFactory` @@ -154,5 +155,98 @@ def test_use_default_due_to_invalid_direction(self): ) +class CPLEXShellSolvePrioritiesFile(unittest.TestCase): + def setUp(self): + from pyomo.solvers.plugins.converter.model import PyomoMIPConverter # register the `ProblemConverterFactory` + from pyomo.repn.plugins.cpxlp import ProblemWriter_cpxlp # register the `WriterFactory` + + def get_mock_model_with_priorities(self): + m = ConcreteModel() + m.x = Var(domain=Integers) + m.s = RangeSet(10) + m.y = Var(m.s, domain=Integers) + m.o = Objective(expr=m.x + sum(m.y), sense=minimize) + m.c = Constraint(expr=m.x >= 1) + m.c2 = Constraint(expr=quicksum(m.y[i] for i in m.s) >= 10) + + m.x.branch_priority = 1 + + for var in m.y.values(): + var.branch_priority = 2 + var.branch_direction = -1 + + m.y[10].branch_direction = 1 + return m + + def test_use_variable_priorities(self): + model = self.get_mock_model_with_priorities() + with SolverFactory('cplex', solver_io='lp') as opt: + opt.solve(model, priorities=True, keepfiles=True) + + with open(opt._priorities_file_name, 'r') as ord_file: + priorities_file = ord_file.read() + + assert priorities_file == ( + '* ENCODING=ISO-8859-1\n' + 'NAME Priority Order\n' + ' x1 1\n' + ' DN x2 2\n' + ' DN x3 2\n' + ' DN x4 2\n' + ' DN x5 2\n' + ' DN x6 2\n' + ' DN x7 2\n' + ' DN x8 2\n' + ' DN x9 2\n' + ' DN x10 2\n' + ' UP x11 2\n' + 'ENDATA\n' + ) + + assert "read %s\n" % (opt._priorities_file_name,) in opt._command.script + + def test_ignore_variable_priorities(self): + model = self.get_mock_model_with_priorities() + with SolverFactory('cplex', solver_io='lp') as opt: + opt.solve(model, priorities=False, keepfiles=True) + + assert opt._priorities_file_name is None + assert ".ord" not in opt._command.script + + def test_can_use_manual_priorities_file_with_lp_solve(self): + """ Test that we can pass an LP file (not a pyomo model) along with a priorities file to `.solve()` """ + model = self.get_mock_model_with_priorities() + + with SolverFactory('cplex', solver_io='lp') as pre_opt: + pre_opt._presolve(model, priorities=True, keepfiles=True) + lp_file = pre_opt._problem_files[0] + priorities_file_name = pre_opt._priorities_file_name + + with SolverFactory('cplex', solver_io='lp') as opt: + opt.solve(lp_file, priorities=True, priorities_file=priorities_file_name, keepfiles=True) + + assert ".ord" in opt._command.script + + with open(opt._priorities_file_name, 'r') as ord_file: + priorities_file = ord_file.read() + + assert priorities_file == ( + '* ENCODING=ISO-8859-1\n' + 'NAME Priority Order\n' + ' x1 1\n' + ' DN x2 2\n' + ' DN x3 2\n' + ' DN x4 2\n' + ' DN x5 2\n' + ' DN x6 2\n' + ' DN x7 2\n' + ' DN x8 2\n' + ' DN x9 2\n' + ' DN x10 2\n' + ' UP x11 2\n' + 'ENDATA\n' + ) + + if __name__ == "__main__": unittest.main() From 5e44956fe377a514ebebe35cc080071fcb197cea Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Thu, 20 Feb 2020 15:24:03 +0000 Subject: [PATCH 03/19] :hammer: Use `Suffix`es instead of `_GeneralVarData` - Declare `priority` and `direction` suffixes on the model instead of attributes on the variable data in order to represent branching decisions --- pyomo/core/base/var.py | 10 +---- pyomo/core/tests/unit/test_var.py | 18 --------- pyomo/solvers/plugins/solvers/CPLEX.py | 37 +++++++++++++---- pyomo/solvers/tests/checks/test_cplex.py | 51 +++++++++++++----------- 4 files changed, 57 insertions(+), 59 deletions(-) diff --git a/pyomo/core/base/var.py b/pyomo/core/base/var.py index b372d7c1183..82e5c788c20 100644 --- a/pyomo/core/base/var.py +++ b/pyomo/core/base/var.py @@ -11,7 +11,6 @@ __all__ = ['Var', '_VarData', '_GeneralVarData', 'VarList', 'SimpleVar'] import logging -from typing import Optional from weakref import ref as weakref_ref from pyomo.common.modeling import NoArgumentGiven @@ -279,10 +278,6 @@ def to_string(self, verbose=None, labeler=None, smap=None, compute_values=False) return self.name -PriorityType = int -BranchDirectionType = int - - class _GeneralVarData(_VarData): """ This class defines the data for a single variable. @@ -313,7 +308,7 @@ class _GeneralVarData(_VarData): these attributes in certain cases. """ - __slots__ = ('_value', '_lb', '_ub', '_domain', 'fixed', 'stale', 'branch_priority', 'branch_direction') + __slots__ = ('_value', '_lb', '_ub', '_domain', 'fixed', 'stale') def __init__(self, domain=Reals, component=None): # @@ -346,9 +341,6 @@ def __init__(self, domain=Reals, component=None): "for bounds (like a Pyomo Set). Examples: NonNegativeReals, " "Integers, Binary" % (domain, (RealSet, IntegerSet, BooleanSet))) - self.branch_priority = None # type: Optional[PriorityType] - self.branch_direction = None # type: Optional[BranchDirectionType] - def __getstate__(self): state = super(_GeneralVarData, self).__getstate__() for i in _GeneralVarData.__slots__: diff --git a/pyomo/core/tests/unit/test_var.py b/pyomo/core/tests/unit/test_var.py index fe814428867..5faf0c44ef8 100644 --- a/pyomo/core/tests/unit/test_var.py +++ b/pyomo/core/tests/unit/test_var.py @@ -440,24 +440,6 @@ def test_value(self): self.assertEqual( type(tmp), int) self.assertEqual( tmp, 3 ) - def test_branch_priority_attr(self): - """Test branch_priority attribute""" - self.model.x = Var() - self.instance = self.model.create_instance() - - self.assertIsNone(self.instance.x.branch_priority) - self.instance.x.branch_priority = 1 - self.assertEqual(self.instance.x.branch_priority, 1) - - def test_branch_direction_attr(self): - """Test branch_direction attribute""" - self.model.x = Var() - self.instance = self.model.create_instance() - - self.assertIsNone(self.instance.x.branch_direction) - self.instance.x.branch_direction = -1 - self.assertEqual(self.instance.x.branch_direction, -1) - class TestArrayVar(TestSimpleVar): diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 015e0938449..04184accc80 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -18,6 +18,7 @@ import pyutilib.common import pyutilib.misc +from pyomo.core import ComponentMap, Suffix, active_export_suffix_generator from pyomo.opt.base import * from pyomo.opt.base.solvers import _extract_version from pyomo.opt.results import * @@ -122,10 +123,9 @@ class CPLEXBranchDirection: @staticmethod def to_str(branch_direction): try: - return { - CPLEXBranchDirection.down: "DN", - CPLEXBranchDirection.up: "UP", - }[branch_direction] + return {CPLEXBranchDirection.down: "DN", CPLEXBranchDirection.up: "UP"}[ + branch_direction + ] except KeyError: return "" @@ -136,7 +136,11 @@ class ORDFileSchema: @classmethod def ROW(cls, name, priority, branch_direction=None): - return " %s %s %s\n" % (CPLEXBranchDirection.to_str(branch_direction), name, priority) + return " %s %s %s\n" % ( + CPLEXBranchDirection.to_str(branch_direction), + name, + priority, + ) @SolverFactory.register('_cplex_shell', doc='Shell interface to the CPLEX LP/MIP solver') @@ -228,20 +232,37 @@ def _warm_start(self, instance): mst_file.write("\n") mst_file.write("\n") + # Expected names of `Suffix` components for branching priorities and directions respectively + SUFFIX_PRIORITY_NAME = "priority" + SUFFIX_DIRECTION_NAME = "direction" + def _write_priorities_file(self, instance) -> None: """ Write a variable priorities file in the CPLEX ORD format. """ from pyomo.core.base import Var if isinstance(instance, IBlock): smap = getattr(instance, "._symbol_maps")[self._smap_id] + suffixes = pyomo.core.kernel.suffix.export_suffix_generator( + instance, datatype=Suffix.INT, active=True, descend_into=False + ) else: smap = instance.solutions.symbol_map[self._smap_id] + suffixes = active_export_suffix_generator(instance, datatype=Suffix.INT) byObject = smap.byObject + suffixes = dict(suffixes) + + if self.SUFFIX_PRIORITY_NAME not in suffixes: + raise ValueError( + "Cannot write branching priorities file as `model.%s` Suffix has not been declared." + % (self.SUFFIX_PRIORITY_NAME,) + ) + + priorities = suffixes[self.SUFFIX_PRIORITY_NAME] + directions = suffixes.get(self.SUFFIX_DIRECTION_NAME, ComponentMap()) with open(self._priorities_file_name, "w") as ord_file: ord_file.write(ORDFileSchema.HEADER) - for var in instance.component_data_objects(Var): - priority = var.branch_priority + for var, priority in priorities.items(): if priority is None: continue @@ -252,7 +273,7 @@ def _write_priorities_file(self, instance) -> None: continue ord_file.write( - ORDFileSchema.ROW(byObject[id(var)], priority, var.branch_direction) + ORDFileSchema.ROW(byObject[id(var)], priority, directions.get(var)) ) ord_file.write(ORDFileSchema.FOOTER) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index f3e5106072c..9ab614c8a42 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -14,7 +14,7 @@ import pyutilib import pyutilib.th as unittest -from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum +from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum, Suffix from pyomo.opt import ProblemFormat, convert_problem, SolverFactory from pyomo.solvers.plugins.solvers.CPLEX import (CPLEXSHELL, MockCPLEX, _validate_file_name) @@ -98,19 +98,13 @@ def get_priorities_file_as_string(self, mock_cplex_shell): priorities_file = ord_file.read() return priorities_file - def test_write_empty_priorities_file(self): - CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) - - with open(self.mock_cplex_shell._priorities_file_name, "r") as ord_file: - priorities_file = ord_file.read() - - self.assertEqual( - priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\nENDATA\n", - ) + def test_write_without_priority_suffix(self): + with self.assertRaises(ValueError): + CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) def test_write_priority_to_priorities_file(self): - self.mock_model.x.branch_priority = 10 + self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority.set_value(self.mock_model.x, 10) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) @@ -119,11 +113,13 @@ def test_write_priority_to_priorities_file(self): priorities_file, "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n", ) - self.assertIsNone(self.mock_model.x.branch_direction) def test_write_priority_and_direction_to_priorities_file(self): - self.mock_model.x.branch_priority = 10 - self.mock_model.x.branch_direction = -1 + self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority.set_value(self.mock_model.x, 10) + + self.mock_model.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.direction.set_value(self.mock_model.x, -1) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) @@ -134,17 +130,21 @@ def test_write_priority_and_direction_to_priorities_file(self): ) def test_raise_due_to_invalid_priority(self): - self.mock_model.x.branch_priority = -1 + self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority.set_value(self.mock_model.x, -1) with self.assertRaises(ValueError): CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) - self.mock_model.x.branch_priority = 1.1 + self.mock_model.priority.set_value(self.mock_model.x, 1.1) with self.assertRaises(ValueError): CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) def test_use_default_due_to_invalid_direction(self): - self.mock_model.x.branch_priority = 10 - self.mock_model.x.branch_direction = "invalid_branching_direction" + self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority.set_value(self.mock_model.x, 10) + + self.mock_model.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.direction.set_value(self.mock_model.x, "invalid_branching_direction") CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) @@ -169,13 +169,16 @@ def get_mock_model_with_priorities(self): m.c = Constraint(expr=m.x >= 1) m.c2 = Constraint(expr=quicksum(m.y[i] for i in m.s) >= 10) - m.x.branch_priority = 1 + m.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + m.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + + m.priority.set_value(m.x, 1) - for var in m.y.values(): - var.branch_priority = 2 - var.branch_direction = -1 + # Ensure tests work for both options of `expand` + m.priority.set_value(m.y, 2, expand=False) + m.direction.set_value(m.y, -1, expand=True) - m.y[10].branch_direction = 1 + m.direction.set_value(m.y[10], 1) return m def test_use_variable_priorities(self): From c4b9c53ff0886897e486cba3038d06c2a0ead64d Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Thu, 20 Feb 2020 15:32:45 +0000 Subject: [PATCH 04/19] :hammer: Allow indexed variables to have separate directions --- pyomo/solvers/plugins/solvers/CPLEX.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 04184accc80..30123ed9e7b 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -263,18 +263,23 @@ def _write_priorities_file(self, instance) -> None: with open(self._priorities_file_name, "w") as ord_file: ord_file.write(ORDFileSchema.HEADER) for var, priority in priorities.items(): - if priority is None: + if priority is None or not var.active: continue if not (0 <= priority == int(priority)): raise ValueError("`priority` must be a non-negative integer") - if id(var) not in byObject or not var.active: - continue + var_direction = directions.get(var, CPLEXBranchDirection.default) + + for child_var in var.values(): + if id(child_var) not in byObject: + continue - ord_file.write( - ORDFileSchema.ROW(byObject[id(var)], priority, directions.get(var)) - ) + child_var_direction = directions.get(child_var, var_direction) + + ord_file.write( + ORDFileSchema.ROW(byObject[id(child_var)], priority, child_var_direction) + ) ord_file.write(ORDFileSchema.FOOTER) From b468b2b692d185ff888d10b42d4ea940336abf45 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Thu, 20 Feb 2020 15:52:48 +0000 Subject: [PATCH 05/19] :bug: Handle both non-indexed vars in Suffix --- pyomo/solvers/plugins/solvers/CPLEX.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 30123ed9e7b..4db1ca59077 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -271,6 +271,15 @@ def _write_priorities_file(self, instance) -> None: var_direction = directions.get(var, CPLEXBranchDirection.default) + if not var.is_indexed(): + if id(var) not in byObject: + continue + + ord_file.write( + ORDFileSchema.ROW(byObject[id(var)], priority, var_direction) + ) + continue + for child_var in var.values(): if id(child_var) not in byObject: continue From 495776d2c4d3bf412ac895ec108a9e7670295dd4 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 21 Feb 2020 08:48:42 +0000 Subject: [PATCH 06/19] :rotating_light: Update tests to work when CPLEX is unavailable --- pyomo/solvers/tests/checks/test_cplex.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 9ab614c8a42..85e57829eda 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -183,8 +183,8 @@ def get_mock_model_with_priorities(self): def test_use_variable_priorities(self): model = self.get_mock_model_with_priorities() - with SolverFactory('cplex', solver_io='lp') as opt: - opt.solve(model, priorities=True, keepfiles=True) + with SolverFactory('_mock_cplex') as opt: + opt._presolve(model, priorities=True, keepfiles=True) with open(opt._priorities_file_name, 'r') as ord_file: priorities_file = ord_file.read() @@ -210,8 +210,8 @@ def test_use_variable_priorities(self): def test_ignore_variable_priorities(self): model = self.get_mock_model_with_priorities() - with SolverFactory('cplex', solver_io='lp') as opt: - opt.solve(model, priorities=False, keepfiles=True) + with SolverFactory('_mock_cplex') as opt: + opt._presolve(model, priorities=True, keepfiles=True) assert opt._priorities_file_name is None assert ".ord" not in opt._command.script @@ -220,13 +220,13 @@ def test_can_use_manual_priorities_file_with_lp_solve(self): """ Test that we can pass an LP file (not a pyomo model) along with a priorities file to `.solve()` """ model = self.get_mock_model_with_priorities() - with SolverFactory('cplex', solver_io='lp') as pre_opt: + with SolverFactory('_mock_cplex') as pre_opt: pre_opt._presolve(model, priorities=True, keepfiles=True) lp_file = pre_opt._problem_files[0] priorities_file_name = pre_opt._priorities_file_name - with SolverFactory('cplex', solver_io='lp') as opt: - opt.solve(lp_file, priorities=True, priorities_file=priorities_file_name, keepfiles=True) + with SolverFactory('_mock_cplex') as opt: + opt._presolve(lp_file, priorities=True, priorities_file=priorities_file_name, keepfiles=True) assert ".ord" in opt._command.script From dbccdf28ec015dfddef38ae386fee533c9c28c57 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 21 Feb 2020 08:50:37 +0000 Subject: [PATCH 07/19] :bug: Fix test --- pyomo/solvers/tests/checks/test_cplex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 85e57829eda..7e7c7830fb9 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -211,7 +211,7 @@ def test_use_variable_priorities(self): def test_ignore_variable_priorities(self): model = self.get_mock_model_with_priorities() with SolverFactory('_mock_cplex') as opt: - opt._presolve(model, priorities=True, keepfiles=True) + opt._presolve(model, priorities=False, keepfiles=True) assert opt._priorities_file_name is None assert ".ord" not in opt._command.script From ba38d43c91e50b28ea405b59713881ed4ff1b855 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 21 Feb 2020 10:02:37 +0000 Subject: [PATCH 08/19] :bug: Fix imports - Looks like there's some circular imports failing specifically for Py2.7 - Follow the same (hacky) local import design chosen in 0935e75e --- pyomo/solvers/plugins/solvers/CPLEX.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 4db1ca59077..2d3418716c8 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -18,7 +18,6 @@ import pyutilib.common import pyutilib.misc -from pyomo.core import ComponentMap, Suffix, active_export_suffix_generator from pyomo.opt.base import * from pyomo.opt.base.solvers import _extract_version from pyomo.opt.results import * @@ -238,7 +237,7 @@ def _warm_start(self, instance): def _write_priorities_file(self, instance) -> None: """ Write a variable priorities file in the CPLEX ORD format. """ - from pyomo.core.base import Var + from pyomo.core.base import Var, ComponentMap, Suffix, active_export_suffix_generator if isinstance(instance, IBlock): smap = getattr(instance, "._symbol_maps")[self._smap_id] From e4f43f0d533c6ec8fb4acfa34fdd2f1e359cc06c Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 21 Feb 2020 12:15:33 +0000 Subject: [PATCH 09/19] :hammer: Make `BranchDirection` global and simplify `_write_priorities_file()` - Update tests to avoid usage of "magic numbers" and reference `BranchDirection` where possible --- pyomo/opt/base/problem.py | 17 +++- pyomo/solvers/plugins/solvers/CPLEX.py | 80 ++++++++--------- pyomo/solvers/tests/checks/test_cplex.py | 104 +++++++++++------------ 3 files changed, 109 insertions(+), 92 deletions(-) diff --git a/pyomo/opt/base/problem.py b/pyomo/opt/base/problem.py index 5eb242b0a2c..ccd4cd19c03 100644 --- a/pyomo/opt/base/problem.py +++ b/pyomo/opt/base/problem.py @@ -8,7 +8,13 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -__all__ = [ 'AbstractProblemWriter', 'WriterFactory', 'ProblemConfigFactory', 'BaseProblemConfig' ] +__all__ = [ + "AbstractProblemWriter", + "WriterFactory", + "ProblemConfigFactory", + "BaseProblemConfig", + "BranchDirection", +] from pyomo.common import Factory @@ -44,3 +50,12 @@ def __enter__(self): def __exit__(self, t, v, traceback): pass + +class BranchDirection(object): + """ Allowed values for MIP variable branching directions in the `direction` Suffix of a model. """ + + default = 0 + down = -1 + up = 1 + + ALL = {default, down, up} diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 2d3418716c8..3b5e2de24cd 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -23,6 +23,7 @@ from pyomo.opt.results import * from pyomo.opt.solver import * from pyomo.solvers.mockmip import MockMIP +from pyomo.core.base import Var, ComponentMap, Suffix, active_export_suffix_generator from pyomo.core.kernel.block import IBlock logger = logging.getLogger('pyomo.solvers') @@ -112,13 +113,7 @@ def __new__(cls, *args, **kwds): return opt -class CPLEXBranchDirection: - default = 0 - down = -1 - up = 1 - - ALL = {default, down, up} - +class CPLEXBranchDirection(BranchDirection): @staticmethod def to_str(branch_direction): try: @@ -129,7 +124,7 @@ def to_str(branch_direction): return "" -class ORDFileSchema: +class ORDFileSchema(object): HEADER = "* ENCODING=ISO-8859-1\nNAME Priority Order\n" FOOTER = "ENDATA\n" @@ -192,9 +187,6 @@ def warm_start_capable(self): # write a warm-start file in the CPLEX MST format. # def _warm_start(self, instance): - - from pyomo.core.base import Var - # for each variable in the symbol_map, add a child to the # variables element. Both continuous and discrete are accepted # (and required, depending on other options), according to the @@ -235,19 +227,19 @@ def _warm_start(self, instance): SUFFIX_PRIORITY_NAME = "priority" SUFFIX_DIRECTION_NAME = "direction" - def _write_priorities_file(self, instance) -> None: + def _write_priorities_file(self, instance): """ Write a variable priorities file in the CPLEX ORD format. """ - from pyomo.core.base import Var, ComponentMap, Suffix, active_export_suffix_generator + priorities, directions = self._get_suffixes(instance) + rows = self._convert_priorities_to_rows(instance, priorities, directions) + self._write_priority_rows(rows) + def _get_suffixes(self, instance): if isinstance(instance, IBlock): - smap = getattr(instance, "._symbol_maps")[self._smap_id] suffixes = pyomo.core.kernel.suffix.export_suffix_generator( instance, datatype=Suffix.INT, active=True, descend_into=False ) else: - smap = instance.solutions.symbol_map[self._smap_id] suffixes = active_export_suffix_generator(instance, datatype=Suffix.INT) - byObject = smap.byObject suffixes = dict(suffixes) if self.SUFFIX_PRIORITY_NAME not in suffixes: @@ -256,39 +248,49 @@ def _write_priorities_file(self, instance) -> None: % (self.SUFFIX_PRIORITY_NAME,) ) - priorities = suffixes[self.SUFFIX_PRIORITY_NAME] - directions = suffixes.get(self.SUFFIX_DIRECTION_NAME, ComponentMap()) + return ( + suffixes[self.SUFFIX_PRIORITY_NAME], + suffixes.get(self.SUFFIX_DIRECTION_NAME, ComponentMap()), + ) - with open(self._priorities_file_name, "w") as ord_file: - ord_file.write(ORDFileSchema.HEADER) - for var, priority in priorities.items(): - if priority is None or not var.active: - continue + def _convert_priorities_to_rows(self, instance, priorities, directions): + if isinstance(instance, IBlock): + smap = getattr(instance, "._symbol_maps")[self._smap_id] + else: + smap = instance.solutions.symbol_map[self._smap_id] + byObject = smap.byObject - if not (0 <= priority == int(priority)): - raise ValueError("`priority` must be a non-negative integer") + rows = [] + for var, priority in priorities.items(): + if priority is None or not var.active: + continue - var_direction = directions.get(var, CPLEXBranchDirection.default) + if not (0 <= priority == int(priority)): + raise ValueError("`priority` must be a non-negative integer") - if not var.is_indexed(): - if id(var) not in byObject: - continue + var_direction = directions.get(var, CPLEXBranchDirection.default) - ord_file.write( - ORDFileSchema.ROW(byObject[id(var)], priority, var_direction) - ) + if not var.is_indexed(): + if id(var) not in byObject: continue - for child_var in var.values(): - if id(child_var) not in byObject: - continue + rows.append((byObject[id(var)], priority, var_direction)) + continue - child_var_direction = directions.get(child_var, var_direction) + for child_var in var.values(): + if id(child_var) not in byObject: + continue - ord_file.write( - ORDFileSchema.ROW(byObject[id(child_var)], priority, child_var_direction) - ) + child_var_direction = directions.get(child_var, var_direction) + rows.append((byObject[id(child_var)], priority, child_var_direction)) + return rows + + def _write_priority_rows(self, rows): + with open(self._priorities_file_name, "w") as ord_file: + ord_file.write(ORDFileSchema.HEADER) + for var_name, priority, direction in rows: + ord_file.write(ORDFileSchema.ROW(var_name, priority, direction)) ord_file.write(ORDFileSchema.FOOTER) # over-ride presolve to extract the warm-start keyword, if specified. diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 7e7c7830fb9..65193731415 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -9,15 +9,14 @@ # ___________________________________________________________________________ import os -import sys import pyutilib import pyutilib.th as unittest from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum, Suffix -from pyomo.opt import ProblemFormat, convert_problem, SolverFactory +from pyomo.opt import ProblemFormat, convert_problem, SolverFactory, BranchDirection from pyomo.solvers.plugins.solvers.CPLEX import (CPLEXSHELL, MockCPLEX, - _validate_file_name) + _validate_file_name, CPLEXBranchDirection) class _mock_cplex_128(object): @@ -104,29 +103,34 @@ def test_write_without_priority_suffix(self): def test_write_priority_to_priorities_file(self): self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.priority.set_value(self.mock_model.x, 10) + priority_val = 10 + self.mock_model.priority.set_value(self.mock_model.x, priority_val) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n", + "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 %d\nENDATA\n" + % (priority_val,), ) def test_write_priority_and_direction_to_priorities_file(self): self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.priority.set_value(self.mock_model.x, 10) + priority_val = 10 + self.mock_model.priority.set_value(self.mock_model.x, priority_val) self.mock_model.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.direction.set_value(self.mock_model.x, -1) + direction_val = BranchDirection.down + self.mock_model.direction.set_value(self.mock_model.x, direction_val) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n DN x1 10\nENDATA\n", + "* ENCODING=ISO-8859-1\nNAME Priority Order\n %s x1 %d\nENDATA\n" + % (CPLEXBranchDirection.to_str(direction_val), priority_val), ) def test_raise_due_to_invalid_priority(self): @@ -141,24 +145,27 @@ def test_raise_due_to_invalid_priority(self): def test_use_default_due_to_invalid_direction(self): self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.priority.set_value(self.mock_model.x, 10) + priority_val = 10 + self.mock_model.priority.set_value(self.mock_model.x, priority_val) self.mock_model.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.direction.set_value(self.mock_model.x, "invalid_branching_direction") + self.mock_model.direction.set_value( + self.mock_model.x, "invalid_branching_direction" + ) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n", + "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 %d\nENDATA\n" + % (priority_val,), ) class CPLEXShellSolvePrioritiesFile(unittest.TestCase): def setUp(self): - from pyomo.solvers.plugins.converter.model import PyomoMIPConverter # register the `ProblemConverterFactory` - from pyomo.repn.plugins.cpxlp import ProblemWriter_cpxlp # register the `WriterFactory` + pass def get_mock_model_with_priorities(self): m = ConcreteModel() @@ -176,41 +183,41 @@ def get_mock_model_with_priorities(self): # Ensure tests work for both options of `expand` m.priority.set_value(m.y, 2, expand=False) - m.direction.set_value(m.y, -1, expand=True) + m.direction.set_value(m.y, BranchDirection.down, expand=True) - m.direction.set_value(m.y[10], 1) + m.direction.set_value(m.y[10], BranchDirection.up) return m def test_use_variable_priorities(self): model = self.get_mock_model_with_priorities() - with SolverFactory('_mock_cplex') as opt: + with SolverFactory("_mock_cplex") as opt: opt._presolve(model, priorities=True, keepfiles=True) - with open(opt._priorities_file_name, 'r') as ord_file: + with open(opt._priorities_file_name, "r") as ord_file: priorities_file = ord_file.read() assert priorities_file == ( - '* ENCODING=ISO-8859-1\n' - 'NAME Priority Order\n' - ' x1 1\n' - ' DN x2 2\n' - ' DN x3 2\n' - ' DN x4 2\n' - ' DN x5 2\n' - ' DN x6 2\n' - ' DN x7 2\n' - ' DN x8 2\n' - ' DN x9 2\n' - ' DN x10 2\n' - ' UP x11 2\n' - 'ENDATA\n' + "* ENCODING=ISO-8859-1\n" + "NAME Priority Order\n" + " x1 1\n" + " DN x2 2\n" + " DN x3 2\n" + " DN x4 2\n" + " DN x5 2\n" + " DN x6 2\n" + " DN x7 2\n" + " DN x8 2\n" + " DN x9 2\n" + " DN x10 2\n" + " UP x11 2\n" + "ENDATA\n" ) assert "read %s\n" % (opt._priorities_file_name,) in opt._command.script def test_ignore_variable_priorities(self): model = self.get_mock_model_with_priorities() - with SolverFactory('_mock_cplex') as opt: + with SolverFactory("_mock_cplex") as opt: opt._presolve(model, priorities=False, keepfiles=True) assert opt._priorities_file_name is None @@ -220,35 +227,28 @@ def test_can_use_manual_priorities_file_with_lp_solve(self): """ Test that we can pass an LP file (not a pyomo model) along with a priorities file to `.solve()` """ model = self.get_mock_model_with_priorities() - with SolverFactory('_mock_cplex') as pre_opt: + with SolverFactory("_mock_cplex") as pre_opt: pre_opt._presolve(model, priorities=True, keepfiles=True) lp_file = pre_opt._problem_files[0] priorities_file_name = pre_opt._priorities_file_name - with SolverFactory('_mock_cplex') as opt: - opt._presolve(lp_file, priorities=True, priorities_file=priorities_file_name, keepfiles=True) + with open(priorities_file_name, "r") as ord_file: + provided_priorities_file = ord_file.read() + + with SolverFactory("_mock_cplex") as opt: + opt._presolve( + lp_file, + priorities=True, + priorities_file=priorities_file_name, + keepfiles=True, + ) assert ".ord" in opt._command.script - with open(opt._priorities_file_name, 'r') as ord_file: + with open(opt._priorities_file_name, "r") as ord_file: priorities_file = ord_file.read() - assert priorities_file == ( - '* ENCODING=ISO-8859-1\n' - 'NAME Priority Order\n' - ' x1 1\n' - ' DN x2 2\n' - ' DN x3 2\n' - ' DN x4 2\n' - ' DN x5 2\n' - ' DN x6 2\n' - ' DN x7 2\n' - ' DN x8 2\n' - ' DN x9 2\n' - ' DN x10 2\n' - ' UP x11 2\n' - 'ENDATA\n' - ) + assert priorities_file == provided_priorities_file if __name__ == "__main__": From ae59e09fa7ecc3c71990b963c985f48901f9b6b5 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 21 Feb 2020 15:20:34 +0000 Subject: [PATCH 10/19] :hammer: Remove unnecessary `CPLEXBranchDirection` --- pyomo/solvers/plugins/solvers/CPLEX.py | 24 +++++++++++------------- pyomo/solvers/tests/checks/test_cplex.py | 5 ++--- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 3b5e2de24cd..6a13c9e0899 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -113,17 +113,6 @@ def __new__(cls, *args, **kwds): return opt -class CPLEXBranchDirection(BranchDirection): - @staticmethod - def to_str(branch_direction): - try: - return {CPLEXBranchDirection.down: "DN", CPLEXBranchDirection.up: "UP"}[ - branch_direction - ] - except KeyError: - return "" - - class ORDFileSchema(object): HEADER = "* ENCODING=ISO-8859-1\nNAME Priority Order\n" FOOTER = "ENDATA\n" @@ -131,11 +120,20 @@ class ORDFileSchema(object): @classmethod def ROW(cls, name, priority, branch_direction=None): return " %s %s %s\n" % ( - CPLEXBranchDirection.to_str(branch_direction), + cls._direction_to_str(branch_direction), name, priority, ) + @staticmethod + def _direction_to_str(branch_direction): + try: + return {BranchDirection.down: "DN", BranchDirection.up: "UP"}[ + branch_direction + ] + except KeyError: + return "" + @SolverFactory.register('_cplex_shell', doc='Shell interface to the CPLEX LP/MIP solver') class CPLEXSHELL(ILMLicensedSystemCallSolver): @@ -268,7 +266,7 @@ def _convert_priorities_to_rows(self, instance, priorities, directions): if not (0 <= priority == int(priority)): raise ValueError("`priority` must be a non-negative integer") - var_direction = directions.get(var, CPLEXBranchDirection.default) + var_direction = directions.get(var, BranchDirection.default) if not var.is_indexed(): if id(var) not in byObject: diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 65193731415..4fc98c581af 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -15,8 +15,7 @@ from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum, Suffix from pyomo.opt import ProblemFormat, convert_problem, SolverFactory, BranchDirection -from pyomo.solvers.plugins.solvers.CPLEX import (CPLEXSHELL, MockCPLEX, - _validate_file_name, CPLEXBranchDirection) +from pyomo.solvers.plugins.solvers.CPLEX import CPLEXSHELL, MockCPLEX, _validate_file_name, ORDFileSchema class _mock_cplex_128(object): @@ -130,7 +129,7 @@ def test_write_priority_and_direction_to_priorities_file(self): self.assertEqual( priorities_file, "* ENCODING=ISO-8859-1\nNAME Priority Order\n %s x1 %d\nENDATA\n" - % (CPLEXBranchDirection.to_str(direction_val), priority_val), + % (ORDFileSchema._direction_to_str(direction_val), priority_val), ) def test_raise_due_to_invalid_priority(self): From 65762690a8fe20af4c7c4f6616328adc1e9f54ec Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Mon, 24 Feb 2020 09:04:48 +0000 Subject: [PATCH 11/19] :hammer: Use `TestCase` assert methods --- pyomo/solvers/tests/checks/test_cplex.py | 44 +++++++++++++----------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 4fc98c581af..2a35413ad51 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -195,32 +195,34 @@ def test_use_variable_priorities(self): with open(opt._priorities_file_name, "r") as ord_file: priorities_file = ord_file.read() - assert priorities_file == ( - "* ENCODING=ISO-8859-1\n" - "NAME Priority Order\n" - " x1 1\n" - " DN x2 2\n" - " DN x3 2\n" - " DN x4 2\n" - " DN x5 2\n" - " DN x6 2\n" - " DN x7 2\n" - " DN x8 2\n" - " DN x9 2\n" - " DN x10 2\n" - " UP x11 2\n" - "ENDATA\n" + self.assertEqual( + priorities_file, + ( + "* ENCODING=ISO-8859-1\n" + "NAME Priority Order\n" + " x1 1\n" + " DN x2 2\n" + " DN x3 2\n" + " DN x4 2\n" + " DN x5 2\n" + " DN x6 2\n" + " DN x7 2\n" + " DN x8 2\n" + " DN x9 2\n" + " DN x10 2\n" + " UP x11 2\n" + "ENDATA\n" + ), ) - - assert "read %s\n" % (opt._priorities_file_name,) in opt._command.script + self.assertIn("read %s\n" % (opt._priorities_file_name,), opt._command.script) def test_ignore_variable_priorities(self): model = self.get_mock_model_with_priorities() with SolverFactory("_mock_cplex") as opt: opt._presolve(model, priorities=False, keepfiles=True) - assert opt._priorities_file_name is None - assert ".ord" not in opt._command.script + self.assertIsNone(opt._priorities_file_name) + self.assertNotIn(".ord", opt._command.script) def test_can_use_manual_priorities_file_with_lp_solve(self): """ Test that we can pass an LP file (not a pyomo model) along with a priorities file to `.solve()` """ @@ -242,12 +244,12 @@ def test_can_use_manual_priorities_file_with_lp_solve(self): keepfiles=True, ) - assert ".ord" in opt._command.script + self.assertIn(".ord", opt._command.script) with open(opt._priorities_file_name, "r") as ord_file: priorities_file = ord_file.read() - assert priorities_file == provided_priorities_file + self.assertEqual(priorities_file, provided_priorities_file) if __name__ == "__main__": From 3eac4cc725ee8fd8c1f5e2f5b404131f230727d0 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Thu, 12 Mar 2020 09:15:09 +0000 Subject: [PATCH 12/19] :rotating_light: Register factories using top-level import --- pyomo/solvers/tests/checks/test_cplex.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 2a35413ad51..8a8a4aeeced 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -13,6 +13,7 @@ import pyutilib import pyutilib.th as unittest +from pyomo.environ import * from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum, Suffix from pyomo.opt import ProblemFormat, convert_problem, SolverFactory, BranchDirection from pyomo.solvers.plugins.solvers.CPLEX import CPLEXSHELL, MockCPLEX, _validate_file_name, ORDFileSchema @@ -62,9 +63,6 @@ def test_validate_file_name(self): class CPLEXShellWritePrioritiesFile(unittest.TestCase): def setUp(self): - from pyomo.solvers.plugins.converter.model import PyomoMIPConverter # register the `ProblemConverterFactory` - from pyomo.repn.plugins.cpxlp import ProblemWriter_cpxlp # register the `WriterFactory` - self.mock_model = self.get_mock_model() self.mock_cplex_shell = self.get_mock_cplex_shell(self.mock_model) self.mock_cplex_shell._priorities_file_name = pyutilib.services.TempfileManager.create_tempfile( @@ -163,9 +161,6 @@ def test_use_default_due_to_invalid_direction(self): class CPLEXShellSolvePrioritiesFile(unittest.TestCase): - def setUp(self): - pass - def get_mock_model_with_priorities(self): m = ConcreteModel() m.x = Var(domain=Integers) From 41524241c630dd4139f7567606d130b0806a52d9 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Thu, 12 Mar 2020 09:15:26 +0000 Subject: [PATCH 13/19] :rotating_light: Hardcode string results --- pyomo/solvers/tests/checks/test_cplex.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 8a8a4aeeced..9dea5137f47 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -16,7 +16,7 @@ from pyomo.environ import * from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum, Suffix from pyomo.opt import ProblemFormat, convert_problem, SolverFactory, BranchDirection -from pyomo.solvers.plugins.solvers.CPLEX import CPLEXSHELL, MockCPLEX, _validate_file_name, ORDFileSchema +from pyomo.solvers.plugins.solvers.CPLEX import CPLEXSHELL, MockCPLEX, _validate_file_name class _mock_cplex_128(object): @@ -126,8 +126,7 @@ def test_write_priority_and_direction_to_priorities_file(self): self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n %s x1 %d\nENDATA\n" - % (ORDFileSchema._direction_to_str(direction_val), priority_val), + "* ENCODING=ISO-8859-1\nNAME Priority Order\n DN x1 10\nENDATA\n" ) def test_raise_due_to_invalid_priority(self): @@ -155,8 +154,7 @@ def test_use_default_due_to_invalid_direction(self): self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 %d\nENDATA\n" - % (priority_val,), + "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n" ) From 37d1c77d76e887e9a4225633d6da8421bbcdd6b3 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 13 Mar 2020 09:09:34 +0000 Subject: [PATCH 14/19] :books: Formatting of test results --- pyomo/solvers/tests/checks/test_cplex.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 9dea5137f47..2bf4db71518 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -108,8 +108,10 @@ def test_write_priority_to_priorities_file(self): self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 %d\nENDATA\n" - % (priority_val,), + "* ENCODING=ISO-8859-1\n" + "NAME Priority Order\n" + " x1 10\n" + "ENDATA\n" ) def test_write_priority_and_direction_to_priorities_file(self): @@ -126,7 +128,10 @@ def test_write_priority_and_direction_to_priorities_file(self): self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n DN x1 10\nENDATA\n" + "* ENCODING=ISO-8859-1\n" + "NAME Priority Order\n" + " DN x1 10\n" + "ENDATA\n" ) def test_raise_due_to_invalid_priority(self): @@ -154,7 +159,10 @@ def test_use_default_due_to_invalid_direction(self): self.assertEqual( priorities_file, - "* ENCODING=ISO-8859-1\nNAME Priority Order\n x1 10\nENDATA\n" + "* ENCODING=ISO-8859-1\n" + "NAME Priority Order\n" + " x1 10\n" + "ENDATA\n" ) From e6d9fcc209530e8ad1fe1d46fa8f76ad4b141804 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 13 Mar 2020 09:09:43 +0000 Subject: [PATCH 15/19] :books: Test docs --- pyomo/solvers/tests/checks/test_cplex.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 2bf4db71518..1293e383f32 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -62,6 +62,7 @@ def test_validate_file_name(self): class CPLEXShellWritePrioritiesFile(unittest.TestCase): + """ Unit test on writing of priorities via `CPLEXSHELL._write_priorities_file()` """ def setUp(self): self.mock_model = self.get_mock_model() self.mock_cplex_shell = self.get_mock_cplex_shell(self.mock_model) @@ -167,6 +168,7 @@ def test_use_default_due_to_invalid_direction(self): class CPLEXShellSolvePrioritiesFile(unittest.TestCase): + """ Integration test on the end-to-end application of priorities via the `Suffix` through a `solve()` """ def get_mock_model_with_priorities(self): m = ConcreteModel() m.x = Var(domain=Integers) From 1a0f41be60bcd9ea6b5b8de6c3bdaf3bcba37b44 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Fri, 13 Mar 2020 10:04:27 +0000 Subject: [PATCH 16/19] :bug: Fix branching priorities for Kernel --- pyomo/solvers/plugins/solvers/CPLEX.py | 15 +++-- pyomo/solvers/tests/checks/test_cplex.py | 74 +++++++++++++++++++----- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 6a13c9e0899..4e4b520a574 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -233,8 +233,11 @@ def _write_priorities_file(self, instance): def _get_suffixes(self, instance): if isinstance(instance, IBlock): - suffixes = pyomo.core.kernel.suffix.export_suffix_generator( - instance, datatype=Suffix.INT, active=True, descend_into=False + suffixes = ( + (suf.name, suf) + for suf in pyomo.core.kernel.suffix.export_suffix_generator( + instance, datatype=Suffix.INT, active=True, descend_into=False + ) ) else: suffixes = active_export_suffix_generator(instance, datatype=Suffix.INT) @@ -254,8 +257,12 @@ def _get_suffixes(self, instance): def _convert_priorities_to_rows(self, instance, priorities, directions): if isinstance(instance, IBlock): smap = getattr(instance, "._symbol_maps")[self._smap_id] + var_is_indexed = lambda var: hasattr(var, 'components') + var_iter = lambda var: var.components() else: smap = instance.solutions.symbol_map[self._smap_id] + var_is_indexed = lambda var: var.is_indexed() + var_iter = lambda var: var.values() byObject = smap.byObject rows = [] @@ -268,14 +275,14 @@ def _convert_priorities_to_rows(self, instance, priorities, directions): var_direction = directions.get(var, BranchDirection.default) - if not var.is_indexed(): + if not var_is_indexed(var): if id(var) not in byObject: continue rows.append((byObject[id(var)], priority, var_direction)) continue - for child_var in var.values(): + for child_var in var_iter(var): if id(child_var) not in byObject: continue diff --git a/pyomo/solvers/tests/checks/test_cplex.py b/pyomo/solvers/tests/checks/test_cplex.py index 1293e383f32..a0959fee9f4 100644 --- a/pyomo/solvers/tests/checks/test_cplex.py +++ b/pyomo/solvers/tests/checks/test_cplex.py @@ -13,8 +13,9 @@ import pyutilib import pyutilib.th as unittest -from pyomo.environ import * +import pyomo.kernel as pmo from pyomo.core import Binary, ConcreteModel, Constraint, Objective, Var, Integers, RangeSet, minimize, quicksum, Suffix +from pyomo.environ import * from pyomo.opt import ProblemFormat, convert_problem, SolverFactory, BranchDirection from pyomo.solvers.plugins.solvers.CPLEX import CPLEXSHELL, MockCPLEX, _validate_file_name @@ -63,6 +64,8 @@ def test_validate_file_name(self): class CPLEXShellWritePrioritiesFile(unittest.TestCase): """ Unit test on writing of priorities via `CPLEXSHELL._write_priorities_file()` """ + suffix_cls = Suffix + def setUp(self): self.mock_model = self.get_mock_model() self.mock_cplex_shell = self.get_mock_cplex_shell(self.mock_model) @@ -95,14 +98,18 @@ def get_priorities_file_as_string(self, mock_cplex_shell): priorities_file = ord_file.read() return priorities_file + @staticmethod + def _set_suffix_value(suffix, variable, value): + suffix.set_value(variable, value) + def test_write_without_priority_suffix(self): with self.assertRaises(ValueError): CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) def test_write_priority_to_priorities_file(self): - self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority = self.suffix_cls(direction=Suffix.EXPORT, datatype=Suffix.INT) priority_val = 10 - self.mock_model.priority.set_value(self.mock_model.x, priority_val) + self._set_suffix_value(self.mock_model.priority, self.mock_model.x, priority_val) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) @@ -116,13 +123,13 @@ def test_write_priority_to_priorities_file(self): ) def test_write_priority_and_direction_to_priorities_file(self): - self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority = self.suffix_cls(direction=Suffix.EXPORT, datatype=Suffix.INT) priority_val = 10 - self.mock_model.priority.set_value(self.mock_model.x, priority_val) + self._set_suffix_value(self.mock_model.priority, self.mock_model.x, priority_val) - self.mock_model.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.direction = self.suffix_cls(direction=Suffix.EXPORT, datatype=Suffix.INT) direction_val = BranchDirection.down - self.mock_model.direction.set_value(self.mock_model.x, direction_val) + self._set_suffix_value(self.mock_model.direction, self.mock_model.x, direction_val) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) priorities_file = self.get_priorities_file_as_string(self.mock_cplex_shell) @@ -136,23 +143,23 @@ def test_write_priority_and_direction_to_priorities_file(self): ) def test_raise_due_to_invalid_priority(self): - self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.priority.set_value(self.mock_model.x, -1) + self.mock_model.priority = self.suffix_cls(direction=Suffix.EXPORT, datatype=Suffix.INT) + self._set_suffix_value(self.mock_model.priority, self.mock_model.x, -1) with self.assertRaises(ValueError): CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) - self.mock_model.priority.set_value(self.mock_model.x, 1.1) + self._set_suffix_value(self.mock_model.priority, self.mock_model.x, 1.1) with self.assertRaises(ValueError): CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) def test_use_default_due_to_invalid_direction(self): - self.mock_model.priority = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + self.mock_model.priority = self.suffix_cls(direction=Suffix.EXPORT, datatype=Suffix.INT) priority_val = 10 - self.mock_model.priority.set_value(self.mock_model.x, priority_val) + self._set_suffix_value(self.mock_model.priority, self.mock_model.x, priority_val) - self.mock_model.direction = Suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) - self.mock_model.direction.set_value( - self.mock_model.x, "invalid_branching_direction" + self.mock_model.direction = self.suffix_cls(direction=Suffix.EXPORT, datatype=Suffix.INT) + self._set_suffix_value( + self.mock_model.direction, self.mock_model.x, "invalid_branching_direction" ) CPLEXSHELL._write_priorities_file(self.mock_cplex_shell, self.mock_model) @@ -167,6 +174,21 @@ def test_use_default_due_to_invalid_direction(self): ) +class CPLEXShellWritePrioritiesFileKernel(CPLEXShellWritePrioritiesFile): + suffix_cls = pmo.suffix + + @staticmethod + def _set_suffix_value(suffix, variable, value): + suffix[variable] = value + + def get_mock_model(self): + model = pmo.block() + model.x = pmo.variable(domain=Binary) + model.con = pmo.constraint(expr=model.x >= 1) + model.obj = pmo.objective(expr=model.x) + return model + + class CPLEXShellSolvePrioritiesFile(unittest.TestCase): """ Integration test on the end-to-end application of priorities via the `Suffix` through a `solve()` """ def get_mock_model_with_priorities(self): @@ -255,5 +277,27 @@ def test_can_use_manual_priorities_file_with_lp_solve(self): self.assertEqual(priorities_file, provided_priorities_file) +class CPLEXShellSolvePrioritiesFileKernel(CPLEXShellSolvePrioritiesFile): + def get_mock_model_with_priorities(self): + m = pmo.block() + m.x = pmo.variable(domain=Integers) + m.s = range(10) + + m.y = pmo.variable_list(pmo.variable(domain=Integers) for _ in m.s) + + m.o = pmo.objective(expr=m.x + sum(m.y), sense=minimize) + m.c = pmo.constraint(expr=m.x >= 1) + m.c2 = pmo.constraint(expr=quicksum(m.y[i] for i in m.s) >= 10) + + m.priority = pmo.suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + m.direction = pmo.suffix(direction=Suffix.EXPORT, datatype=Suffix.INT) + + m.priority[m.x] = 1 + m.priority[m.y] = 2 + m.direction[m.y] = BranchDirection.down + m.direction[m.y[-1]] = BranchDirection.up + return m + + if __name__ == "__main__": unittest.main() From 67bfca0a6473100791b8c93968fa8eef94e8e530 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Mon, 16 Mar 2020 11:50:56 +0000 Subject: [PATCH 17/19] :hammer: Add `iter_component()` - Add iterator for looping over the data objects of either a `base` or `kernel` component --- pyomo/util/components.py | 30 ++++++++++++++++- pyomo/util/tests/test_components.py | 52 ++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/pyomo/util/components.py b/pyomo/util/components.py index a6a30f8f71d..4c24979a119 100644 --- a/pyomo/util/components.py +++ b/pyomo/util/components.py @@ -8,8 +8,9 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from pyomo.core.kernel.component_map import ComponentMap from pyomo.common.modeling import unique_component_name +from pyomo.core.kernel.component_map import ComponentMap + def rename_components(model, component_list, prefix): """ @@ -52,3 +53,30 @@ def rename_components(model, component_list, prefix): name_map[c] = old_name return name_map + + +def iter_component(obj): + """ + Yield "child" objects from a component that is defined with either the `base` or `kernel` APIs. + If the component is not indexed, it returns itself. + + Parameters + ---------- + obj : ComponentType + eg. `TupleContainer`, `ListContainer`, `DictContainer`, `IndexedComponent`, or `Component` + + Returns + ------- + Iterator[ComponentType] : Iterator of the component data objects. + """ + try: + # catches `IndexedComponent`, and kernel's `_dict` + return iter(obj.values()) + except AttributeError: + pass + + try: + # catches list and kernel's `_list` and `_tuple` + return iter(obj) + except TypeError: + return iter((obj,)) diff --git a/pyomo/util/tests/test_components.py b/pyomo/util/tests/test_components.py index d6c76f035b6..f4a886f155e 100644 --- a/pyomo/util/tests/test_components.py +++ b/pyomo/util/tests/test_components.py @@ -7,10 +7,13 @@ # rights in this software. # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from itertools import zip_longest import pyutilib.th as unittest + import pyomo.environ as pe -from pyomo.util.components import rename_components +import pyomo.kernel as pmo +from pyomo.util.components import iter_component, rename_components class TestUtilComponents(unittest.TestCase): @@ -46,6 +49,53 @@ def con_rule(m, i): self.assertEquals(model.b.name, 'b') self.assertEquals(model.b.scaled_bz.name, 'b.scaled_bz') + def assertSameComponents(self, obj, other_obj): + for i, j in zip_longest(obj, other_obj): + self.assertEqual(id(i), id(j)) + + def test_iter_component_base(self): + model = pe.ConcreteModel() + model.x = pe.Var([1, 2, 3], initialize=0) + model.z = pe.Var(initialize=0) + + def con_rule(m, i): + return m.x[i] + m.z == i + + model.con = pe.Constraint([1, 2, 3], rule=con_rule) + model.zcon = pe.Constraint(expr=model.z >= model.x[2]) + + self.assertSameComponents(list(iter_component(model.x)), list(model.x.values())) + self.assertSameComponents(list(iter_component(model.z)), [model.z[None]]) + self.assertSameComponents( + list(iter_component(model.con)), list(model.con.values()) + ) + self.assertSameComponents(list(iter_component(model.zcon)), [model.zcon[None]]) + + def test_iter_component_kernel(self): + model = pmo.block() + model.x = pmo.variable_list(pmo.variable(value=0) for _ in [1, 2, 3]) + model.z = pmo.variable(value=0) + + model.con = pmo.constraint_dict( + (i, pmo.constraint(expr=model.x[i - 1] + model.z == i)) for i in [1, 2, 3] + ) + model.zcon = pmo.constraint(expr=model.z >= model.x[2]) + + model.param_t = pmo.parameter_tuple(pmo.parameter(value=36) for _ in [1, 2, 3]) + model.param = pmo.parameter(value=42) + + self.assertSameComponents(list(iter_component(model.x)), list(model.x)) + self.assertSameComponents(list(iter_component(model.z)), [model.z]) + self.assertSameComponents( + list(iter_component(model.con)), list(model.con.values()) + ) + self.assertSameComponents(list(iter_component(model.zcon)), [model.zcon]) + self.assertSameComponents( + list(iter_component(model.param_t)), list(model.param_t) + ) + self.assertSameComponents(list(iter_component(model.param)), [model.param]) + + if __name__ == '__main__': # t = TestUtilComponents() # t.test_rename_components() From e663b14b211ba0a82a3df173fc4598a2cbdd1622 Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Mon, 16 Mar 2020 11:51:24 +0000 Subject: [PATCH 18/19] :hammer: Use `iter_component()` --- pyomo/solvers/plugins/solvers/CPLEX.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/CPLEX.py b/pyomo/solvers/plugins/solvers/CPLEX.py index 4e4b520a574..80df23f6bac 100644 --- a/pyomo/solvers/plugins/solvers/CPLEX.py +++ b/pyomo/solvers/plugins/solvers/CPLEX.py @@ -25,6 +25,7 @@ from pyomo.solvers.mockmip import MockMIP from pyomo.core.base import Var, ComponentMap, Suffix, active_export_suffix_generator from pyomo.core.kernel.block import IBlock +from pyomo.util.components import iter_component logger = logging.getLogger('pyomo.solvers') @@ -257,12 +258,8 @@ def _get_suffixes(self, instance): def _convert_priorities_to_rows(self, instance, priorities, directions): if isinstance(instance, IBlock): smap = getattr(instance, "._symbol_maps")[self._smap_id] - var_is_indexed = lambda var: hasattr(var, 'components') - var_iter = lambda var: var.components() else: smap = instance.solutions.symbol_map[self._smap_id] - var_is_indexed = lambda var: var.is_indexed() - var_iter = lambda var: var.values() byObject = smap.byObject rows = [] @@ -275,14 +272,7 @@ def _convert_priorities_to_rows(self, instance, priorities, directions): var_direction = directions.get(var, BranchDirection.default) - if not var_is_indexed(var): - if id(var) not in byObject: - continue - - rows.append((byObject[id(var)], priority, var_direction)) - continue - - for child_var in var_iter(var): + for child_var in iter_component(var): if id(child_var) not in byObject: continue From 456cb8672273d1318f5784bead81a610b5ba1bda Mon Sep 17 00:00:00 2001 From: Ruaridh Williamson Date: Mon, 16 Mar 2020 22:31:41 +0000 Subject: [PATCH 19/19] :bug: Fix Py2 import error --- pyomo/util/tests/test_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/util/tests/test_components.py b/pyomo/util/tests/test_components.py index f4a886f155e..a133e0c841d 100644 --- a/pyomo/util/tests/test_components.py +++ b/pyomo/util/tests/test_components.py @@ -7,7 +7,7 @@ # rights in this software. # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from itertools import zip_longest +from six.moves import zip_longest import pyutilib.th as unittest