diff --git a/CHANGELOG.md b/CHANGELOG.md index 84773e44..e32dfc68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ### User-facing changes +|new| Decision variables and global expressions can have a `title` defined, which will be available in the model results as attributes of those components and can be used for e.g. visualisation (#582). +Parameter titles from the model definition schema will also propagate to the model inputs. + |fixed| Backend parameter updates propagate correctly through global expressions in the order those expressions were defined (#616). |fixed| If setting `model.backend.verbose_strings()`, rebuilt model components from making backend parameter updates will automatically have verbose strings (#623). diff --git a/docs/user_defined_math/components.md b/docs/user_defined_math/components.md index c3f25b44..f45e3e73 100644 --- a/docs/user_defined_math/components.md +++ b/docs/user_defined_math/components.md @@ -16,7 +16,7 @@ variables: ``` 1. It needs a unique name (`storage_cap` in the example above). -1. Ideally, it has a long-form `description` and a `unit` added. +1. Ideally, it has a long-form name (`title`), a `description` and a `unit` added. These are not required, but are useful metadata for later reference. 1. It can have a top-level `foreach` list and `where` string. Without a `foreach`, it becomes an un-indexed variable. @@ -54,7 +54,7 @@ global_expressions: Global expressions are by no means necessary to include, but can make more complex linear expressions easier to keep track of and can reduce post-processing requirements. 1. It needs a unique name (`cost` in the above example). -1. Ideally, it has a long-form `description` and a `unit` added. +1. Ideally, it has a long-form name (`title`), a `description` and a `unit` added. These are not required, but are useful metadata for later reference. 1. It can have a top-level `foreach` list and `where` string. Without a `foreach`, it becomes an un-indexed expression. @@ -78,7 +78,7 @@ constraints: ``` 1. It needs a unique name (`set_storage_initial` in the above example). -1. Ideally, it has a long-form `description` and a `unit` added. +1. Ideally, it has a long-form `description` added. These are not required, but are useful metadata for later reference. 1. It can have a top-level `foreach` list and `where` string. Without a `foreach`, it becomes an un-indexed constraint. diff --git a/src/calliope/backend/backend_model.py b/src/calliope/backend/backend_model.py index 1cbba492..365d4644 100644 --- a/src/calliope/backend/backend_model.py +++ b/src/calliope/backend/backend_model.py @@ -57,8 +57,9 @@ class BackendModelGenerator(ABC): """Helper class for backends.""" _VALID_COMPONENTS: tuple[_COMPONENTS_T, ...] = typing.get_args(_COMPONENTS_T) - _COMPONENT_ATTR_METADATA = ["description", "unit", "default"] + _COMPONENT_ATTR_METADATA = ["description", "unit", "default", "title"] + _PARAM_TITLES = extract_from_schema(MODEL_SCHEMA, "title") _PARAM_DESCRIPTIONS = extract_from_schema(MODEL_SCHEMA, "description") _PARAM_UNITS = extract_from_schema(MODEL_SCHEMA, "x-unit") diff --git a/src/calliope/backend/latex_backend_model.py b/src/calliope/backend/latex_backend_model.py index 89bc1eeb..6ab5a7d0 100644 --- a/src/calliope/backend/latex_backend_model.py +++ b/src/calliope/backend/latex_backend_model.py @@ -375,6 +375,7 @@ def add_parameter( # noqa: D102, override use_inf_as_na: bool = False, ) -> None: attrs = { + "title": self._PARAM_TITLES.get(parameter_name, None), "description": self._PARAM_DESCRIPTIONS.get(parameter_name, None), "unit": self._PARAM_UNITS.get(parameter_name, None), } diff --git a/src/calliope/backend/parsing.py b/src/calliope/backend/parsing.py index b6d96942..9da91db3 100644 --- a/src/calliope/backend/parsing.py +++ b/src/calliope/backend/parsing.py @@ -54,6 +54,7 @@ class UnparsedConstraintDict(TypedDict): class UnparsedExpressionDict(UnparsedConstraintDict): """Unparsed expression checker class.""" + title: NotRequired[str] unit: NotRequired[str] @@ -69,6 +70,7 @@ class UnparsedVariableBoundDict(TypedDict): class UnparsedVariableDict(TypedDict): """Unparsed variable checker class.""" + title: NotRequired[str] description: NotRequired[str] unit: NotRequired[str] foreach: list[str] diff --git a/src/calliope/backend/pyomo_backend_model.py b/src/calliope/backend/pyomo_backend_model.py index a2927352..a63ed204 100644 --- a/src/calliope/backend/pyomo_backend_model.py +++ b/src/calliope/backend/pyomo_backend_model.py @@ -103,6 +103,7 @@ def add_parameter( # noqa: D102, override parameter_da.attrs["original_dtype"] = parameter_values.dtype attrs = { + "title": self._PARAM_TITLES.get(parameter_name, None), "description": self._PARAM_DESCRIPTIONS.get(parameter_name, None), "unit": self._PARAM_UNITS.get(parameter_name, None), "default": default, diff --git a/src/calliope/config/math_schema.yaml b/src/calliope/config/math_schema.yaml index 9a3cf64d..a75b3602 100644 --- a/src/calliope/config/math_schema.yaml +++ b/src/calliope/config/math_schema.yaml @@ -14,6 +14,9 @@ properties: additionalProperties: false required: ["equations"] properties: + title: &title + type: string + description: The component long name, for use in visualisation. description: &description type: string description: A verbose description of the component. @@ -96,6 +99,7 @@ properties: additionalProperties: false required: ["equations"] properties: + title: *title description: *description active: *active unit: &unit @@ -122,6 +126,7 @@ properties: description: A named variable. additionalProperties: false properties: + title: *title description: *description active: *active unit: *unit diff --git a/src/calliope/config/model_def_schema.yaml b/src/calliope/config/model_def_schema.yaml index 500e15be..b9fd2ab9 100644 --- a/src/calliope/config/model_def_schema.yaml +++ b/src/calliope/config/model_def_schema.yaml @@ -314,7 +314,7 @@ properties: type: ["null", string] x-resample_method: first x-type: str - title: Technology longname. + title: Technology long-name. description: Long name of technology, which can be used in post-processing (e.g., plotting). default: .nan diff --git a/src/calliope/math/base.yaml b/src/calliope/math/base.yaml index 86ffdcd8..36ba1daf 100644 --- a/src/calliope/math/base.yaml +++ b/src/calliope/math/base.yaml @@ -489,6 +489,7 @@ constraints: variables: flow_cap: + title: Technology flow (a.k.a. nominal) capacity description: >- A technology's flow capacity, also known as its nominal or nameplate capacity. default: 0 @@ -499,6 +500,7 @@ variables: max: flow_cap_max link_flow_cap: + title: Link flow capacity description: >- A transmission technology's flow capacity, also known as its nominal or nameplate capacity. default: 0 @@ -510,6 +512,7 @@ variables: max: .inf flow_out: + title: Carrier outflow description: >- The outflow of a technology per timestep, also known as the flow discharged (from `storage` technologies) @@ -523,6 +526,7 @@ variables: max: .inf flow_in: + title: Carrier inflow description: >- The inflow to a technology per timestep, also known as the flow consumed (by `storage` technologies) @@ -536,6 +540,7 @@ variables: max: .inf flow_export: + title: Carrier export description: >- The flow of a carrier exported outside the system boundaries by a technology per timestep. default: 0 @@ -547,6 +552,7 @@ variables: max: .inf area_use: + title: Area utilisation description: >- The area in space utilised directly (e.g., solar PV panels) or indirectly (e.g., biofuel crops) by a technology. @@ -559,6 +565,7 @@ variables: max: area_use_max source_use: + title: Source flow use description: >- The carrier flow consumed from outside the system boundaries by a `supply` technology. default: 0 @@ -570,6 +577,7 @@ variables: max: .inf source_cap: + title: Source flow capacity description: >- The upper limit on a flow that can be consumed from outside the system boundaries by a `supply` technology in each timestep. @@ -583,6 +591,7 @@ variables: # --8<-- [start:variable] storage_cap: + title: Stored carrier capacity description: >- The upper limit on a carrier that can be stored by a technology in any timestep. @@ -598,6 +607,7 @@ variables: # --8<-- [end:variable] storage: + title: Stored carrier description: >- The carrier stored by a `storage` technology in each timestep. default: 0 @@ -609,6 +619,7 @@ variables: max: .inf purchased_units: + title: Number of purchased units description: | Integer number of a technology that has been purchased, for any technology set to require integer capacity purchasing. @@ -633,6 +644,7 @@ variables: max: purchased_units_max operating_units: + title: Number of operating units description: >- Integer number of a technology that is operating in each timestep, for any technology set to require integer capacity purchasing. @@ -646,6 +658,7 @@ variables: max: .inf available_flow_cap: + title: Available carrier flow capacity description: >- Flow capacity that will be set to zero if the technology is not operating in a given timestep and will be set to the value of the decision variable `flow_cap` otherwise. @@ -658,6 +671,7 @@ variables: max: .inf async_flow_switch: + title: Asynchronous carrier flow switch description: >- Binary switch to force asynchronous outflow/consumption of technologies with both `flow_in` and `flow_out` defined. @@ -673,6 +687,7 @@ variables: max: 1 unmet_demand: + title: Unmet demand (load shedding) description: >- Virtual source of carrier flow to ensure model feasibility. This should only be considered a debugging rather than a modelling tool as it may @@ -688,6 +703,7 @@ variables: max: .inf unused_supply: + title: Unused supply (curtailment) description: >- Virtual sink of carrier flow to ensure model feasibility. This should only be considered a debugging rather than a modelling tool as it may @@ -743,6 +759,7 @@ objectives: global_expressions: flow_out_inc_eff: + title: Carrier outflow including losses description: >- Outflows after taking efficiency losses into account. default: 0 @@ -759,6 +776,7 @@ global_expressions: expression: flow_out / (flow_out_eff * flow_out_parasitic_eff) flow_in_inc_eff: + title: Carrier inflow including losses description: >- Inflows after taking efficiency losses into account. default: 0 @@ -771,6 +789,7 @@ global_expressions: expression: flow_in * flow_in_eff cost_var: + title: Variable operating costs description: >- The operating costs per timestep of a technology. default: 0 @@ -794,6 +813,7 @@ global_expressions: - expression: sum(cost_flow_out * flow_out, over=carriers) cost_investment_flow_cap: + title: Flow capacity investment costs description: >- The investment costs associated with the nominal/rated capacity of a technology. default: 0 @@ -809,6 +829,7 @@ global_expressions: expression: cost_flow_cap cost_investment_storage_cap: + title: Storage capacity investment costs description: >- The investment costs associated with the storage capacity of a technology. default: 0 @@ -818,6 +839,7 @@ global_expressions: - expression: cost_storage_cap * storage_cap cost_investment_source_cap: + title: Source flow capacity investment costs description: >- The investment costs associated with the source consumption capacity of a technology. default: 0 @@ -827,6 +849,7 @@ global_expressions: - expression: cost_source_cap * source_cap cost_investment_area_use: + title: Area utilisation investment costs description: >- The investment costs associated with the area used by a technology. default: 0 @@ -836,6 +859,7 @@ global_expressions: - expression: cost_area_use * area_use cost_investment_purchase: + title: Binary purchase investment costs description: >- The investment costs associated with the binary purchase of a technology. default: 0 @@ -848,6 +872,7 @@ global_expressions: expression: cost_purchase * purchased_units cost_investment: + title: Total investment costs description: >- The installation costs of a technology, including annualised investment costs and annual maintenance costs. default: 0 @@ -883,6 +908,7 @@ global_expressions: # --8<-- [start:expression] cost: + title: Total costs description: >- The total annualised costs of a technology, including installation and operation costs. diff --git a/src/calliope/math/storage_inter_cluster.yaml b/src/calliope/math/storage_inter_cluster.yaml index ceb0ec0c..1c0165c5 100644 --- a/src/calliope/math/storage_inter_cluster.yaml +++ b/src/calliope/math/storage_inter_cluster.yaml @@ -98,6 +98,7 @@ constraints: variables: storage: + title: Virtual stored carrier description: >- The virtual carrier stored by a `supply_plus` or `storage` technology in each timestep of a clustered day. Stored carrier can be negative so long as it does not go below the carrier stored in `storage_inter_cluster`. @@ -107,6 +108,7 @@ variables: min: -.inf storage_inter_cluster: + title: Virtual inter-cluster stored carrier description: >- The virtual carrier stored by a `supply_plus` or `storage` technology between days of the entire timeseries. Only together with `storage` does this variable's values gain physical significance. @@ -117,6 +119,7 @@ variables: max: .inf storage_intra_cluster_max: + title: Virtual maximum intra-cluster stored carrier description: >- Virtual variable to limit the maximum value of `storage` in a given representative day. unit: energy @@ -127,6 +130,7 @@ variables: max: .inf storage_intra_cluster_min: + title: Virtual minimum intra-cluster stored carrier description: >- Virtual variable to limit the minimum value of `storage` in a given representative day. unit: energy diff --git a/tests/conftest.py b/tests/conftest.py index 37e2d0f3..b1e63b13 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,12 @@ ALL_DIMS = {"nodes", "techs", "carriers", "costs", "timesteps"} +@pytest.fixture(scope="session") +def dummy_int() -> int: + """Dummy integer value that will never be confused by a model value/default.""" + return 0xDEADBEEF + + @pytest.fixture( scope="session", params=set( diff --git a/tests/test_backend_pyomo.py b/tests/test_backend_pyomo.py index ea0fbb0b..2a32770e 100755 --- a/tests/test_backend_pyomo.py +++ b/tests/test_backend_pyomo.py @@ -16,8 +16,6 @@ from .common.util import build_test_model as build_model from .common.util import check_error_or_warning, check_variable_exists -DUMMY_INT = 0xDEADBEEF - @pytest.mark.xfail(reason="Not expecting operate mode to work at the moment") class TestChecks: @@ -1627,10 +1625,10 @@ def simple_supply_longnames(self): @pytest.fixture(scope="class") def simple_supply_updated_cost_flow_cap( - self, simple_supply: calliope.Model + self, simple_supply: calliope.Model, dummy_int: int ) -> calliope.Model: simple_supply.backend.verbose_strings() - simple_supply.backend.update_parameter("cost_flow_cap", DUMMY_INT) + simple_supply.backend.update_parameter("cost_flow_cap", dummy_int) return simple_supply @pytest.fixture() @@ -1706,21 +1704,21 @@ def test_new_build_get_missing_component(self, simple_supply, component_type): getattr(simple_supply.backend, f"get_{component_type}")("foo") def test_new_build_get_variable(self, simple_supply): + """Check a decision variable has the correct data type and has all expected attributes.""" var = simple_supply.backend.get_variable("flow_cap") assert ( var.to_series().dropna().apply(lambda x: isinstance(x, pmo.variable)).all() ) - expected_keys = set( - [ - "obj_type", - "references", - "description", - "unit", - "default", - "yaml_snippet", - "coords_in_name", - ] - ) + expected_keys = { + "obj_type", + "references", + "title", + "description", + "unit", + "default", + "yaml_snippet", + "coords_in_name", + } assert not expected_keys.symmetric_difference(var.attrs.keys()) assert var.attrs["obj_type"] == "variables" assert var.attrs["references"] == { @@ -1743,6 +1741,7 @@ def test_new_build_get_variable_as_vals(self, simple_supply): ) def test_new_build_get_parameter(self, simple_supply): + """Check a parameter has the correct data type and has all expected attributes.""" param = simple_supply.backend.get_parameter("flow_in_eff") assert isinstance(param.item(), pmo.parameter) assert param.attrs == { @@ -1752,6 +1751,7 @@ def test_new_build_get_parameter(self, simple_supply): "references": {"flow_in_inc_eff"}, "coords_in_name": False, "default": 1.0, + "title": "Inflow efficiency", "description": ( "Conversion efficiency from `source`/`flow_in` (tech dependent) into the technology. " "Set as value between 1 (no loss) and 0 (all lost)." @@ -1766,6 +1766,7 @@ def test_new_build_get_parameter_as_vals(self, simple_supply): assert param.dtype == np.dtype("float64") def test_new_build_get_global_expression(self, simple_supply): + """Check a global expression has the correct data type and has all expected attributes.""" expr = simple_supply.backend.get_global_expression("cost_investment") assert ( expr.to_series() @@ -1773,17 +1774,16 @@ def test_new_build_get_global_expression(self, simple_supply): .apply(lambda x: isinstance(x, pmo.expression)) .all() ) - expected_keys = set( - [ - "obj_type", - "references", - "description", - "unit", - "default", - "yaml_snippet", - "coords_in_name", - ] - ) + expected_keys = { + "obj_type", + "references", + "title", + "description", + "unit", + "default", + "yaml_snippet", + "coords_in_name", + } assert not expected_keys.symmetric_difference(expr.attrs.keys()) assert expr.attrs["obj_type"] == "global_expressions" assert expr.attrs["references"] == {"cost"} @@ -1812,9 +1812,14 @@ def test_new_build_get_constraint(self, simple_supply): .apply(lambda x: isinstance(x, pmo.constraint)) .all() ) - expected_keys = set( - ["obj_type", "references", "description", "yaml_snippet", "coords_in_name"] - ) + expected_keys = { + "obj_type", + "references", + "description", + "yaml_snippet", + "coords_in_name", + } + assert not expected_keys.symmetric_difference(constr.attrs.keys()) assert constr.attrs["obj_type"] == "constraints" assert constr.attrs["references"] == set() @@ -2198,8 +2203,8 @@ def test_update_parameter(self, simple_supply): ) assert expected.where(updated_param.notnull()).equals(updated_param) - def test_update_parameter_one_val(self, caplog, simple_supply): - updated_param = DUMMY_INT + def test_update_parameter_one_val(self, caplog, simple_supply, dummy_int: int): + updated_param = dummy_int new_dims = {"techs"} caplog.set_level(logging.DEBUG) @@ -2212,7 +2217,7 @@ def test_update_parameter_one_val(self, caplog, simple_supply): expected = simple_supply.backend.get_parameter( "flow_out_eff", as_backend_objs=False ) - assert (expected == DUMMY_INT).all() + assert (expected == dummy_int).all() def test_update_parameter_replace_defaults(self, simple_supply): updated_param = simple_supply.inputs.flow_out_eff.fillna(0.1) diff --git a/tests/test_core_model.py b/tests/test_core_model.py index 5d0c3e87..ed859719 100644 --- a/tests/test_core_model.py +++ b/tests/test_core_model.py @@ -65,14 +65,26 @@ def test_add_observed_dict_not_dict(self, national_scale_example): class TestAddMath: - @pytest.fixture() + @pytest.fixture(scope="class") def storage_inter_cluster(self): return build_model( {"config.init.add_math": ["storage_inter_cluster"]}, "simple_supply,two_hours,investment_costs", ) - @pytest.fixture() + @pytest.fixture(scope="class") + def storage_inter_cluster_plus_user_def(self, temp_path, dummy_int: int): + new_constraint = calliope.AttrDict( + {"variables": {"storage": {"bounds": {"min": dummy_int}}}} + ) + file_path = temp_path.join("custom-math.yaml") + new_constraint.to_yaml(file_path) + return build_model( + {"config.init.add_math": ["storage_inter_cluster", str(file_path)]}, + "simple_supply,two_hours,investment_costs", + ) + + @pytest.fixture(scope="class") def temp_path(self, tmpdir_factory): return tmpdir_factory.mktemp("custom_math") @@ -179,30 +191,25 @@ def test_override_order(self, temp_path, simple_supply): assert base[i] == new[i] def test_override_existing_internal_constraint_merge( - self, temp_path, simple_supply + self, simple_supply, storage_inter_cluster, storage_inter_cluster_plus_user_def ): - new_constraint = calliope.AttrDict( - {"variables": {"storage": {"bounds": {"min": -1}}}} - ) - file_path = temp_path.join("custom-math.yaml") - new_constraint.to_yaml(file_path) - m = build_model( - {"config.init.add_math": ["storage_inter_cluster", str(file_path)]}, - "simple_supply,two_hours,investment_costs", - ) - base = simple_supply.math["variables"]["storage"] - new = m.math["variables"]["storage"] - - for i in base.keys(): - if i == "bounds": - assert new[i]["min"] == -1 - assert new[i]["max"] == new[i]["max"] - elif i == "description": - assert new[i].startswith( - "The virtual carrier stored by a `supply_plus` or `storage` technology" - ) - else: - assert base[i] == new[i] + storage_inter_cluster_math = storage_inter_cluster.math["variables"]["storage"] + base_math = simple_supply.math["variables"]["storage"] + new_math = storage_inter_cluster_plus_user_def.math["variables"]["storage"] + expected = { + "title": storage_inter_cluster_math["title"], + "description": storage_inter_cluster_math["description"], + "default": base_math["default"], + "unit": base_math["unit"], + "foreach": base_math["foreach"], + "where": base_math["where"], + "bounds": { + "min": new_math["bounds"]["min"], + "max": base_math["bounds"]["max"], + }, + } + + assert new_math == expected class TestValidateMathDict: