Add top-level param tests

src/calliope/preprocess/checks.py

@@ -597,7 +597,10 @@ def check_model_data(model_data):
             model_data.timestep_resolution.loc[i].values
             for i in np.unique(model_data.timesteps.to_index().strftime("%Y-%m-%d"))
         ]
-        if not np.all(daily_timesteps == daily_timesteps[0]):
+        daily_timestep_shapes = set(day.shape for day in daily_timesteps)
+        if len(daily_timestep_shapes) > 1 or not np.all(
+            daily_timesteps == daily_timesteps[0]
+        ):
             model_data.attrs["allow_operate_mode"] = 0
             model_warnings.append(
                 "Operational mode requires the same timestep resolution profile "

src/calliope/preprocess/model_data.py

@@ -104,17 +104,25 @@ def _add_top_level_params(self):
                     f"Trying to add top-level parameter with same name as a node/tech level parameter: {param_name}"
                 )
             if "dims" in param_data:
+                index = param_data.get("index", None)
+                if index is None or not isinstance(index, list):
+                    raise ValueError(
+                        f"(parameters, {param_name}) | Expected list for `index`, received: {index}"
+                    )
+                _index = [
+                    tuple(idx) if isinstance(idx, list) else tuple([idx])
+                    for idx in param_data["index"]
+                ]
+                _dims = (
+                    param_data["dims"]
+                    if isinstance(param_data["dims"], list)
+                    else [param_data["dims"]]
+                )
                 param_series = pd.Series(
                     data=param_data["data"],
-                    index=param_data["index"],
+                    index=pd.MultiIndex.from_tuples(_index, names=_dims),
                     name=param_name,
                 )
-                if isinstance(param_data["dims"], list) and len(param_data["dims"]) > 1:
-                    param_series.index = pd.MultiIndex.from_tuples(
-                        param_series.index, names=param_data["dims"]
-                    )
-                else:
-                    param_series = param_series.rename_axis(param_data["dims"])
                 param_da = param_series.to_xarray()
             else:
                 param_da = xr.DataArray(param_data["data"], name=param_name)
@@ -126,7 +134,7 @@ def _add_top_level_params(self):
                     == "M"
                 ):
                     LOGGER.debug(
-                        f"Updating `{param_name}` {coord_name} dimension index values to datetime format"
+                        f"(parameters, {param_name}) | Updating {coord_name} dimension index values to datetime format"
                     )
                     coords_to_update[coord_name] = pd.to_datetime(
                         coord_data, format="ISO8601"
@@ -137,15 +145,15 @@ def _add_top_level_params(self):
             for coord_name, coord_data in param_da.coords.items():
                 if coord_name not in self.model_data.coords:
                     LOGGER.debug(
-                        f"top-level parameter `{param_name}` is adding a new dimension to the model: {coord_name}"
+                        f"(parameters, {param_name}) | Adding a new dimension to the model: {coord_name}"
                     )
                 else:
                     new_coord_data = coord_data[
                         ~coord_data.isin(self.model_data.coords[coord_name])
                     ]
                     if new_coord_data.size > 0:
                         LOGGER.debug(
-                            f"top-level parameter `{param_name}` is adding a new value to the "
+                            f"(parameters, {param_name}) | Adding a new value to the "
                             f"`{coord_name}` model coordinate: {new_coord_data.values}"
                         )
             self.model_data = self.model_data.merge(param_da.to_dataset())

tests/test_model_data.py

@@ -5,11 +5,13 @@
 import numpy as np
 import pandas as pd
 import pytest
+import xarray as xr
 from calliope._version import __version__
 from calliope.core.attrdict import AttrDict
 from calliope.preprocess import model_run_from_yaml
 from calliope.preprocess.model_data import ModelDataFactory
 
+from .common.util import build_test_model as build_model
 from .common.util import check_error_or_warning
 
 
@@ -413,3 +415,201 @@ def test_add_attributes(self, model_data_w_params):
         attr_dict["calliope_version"] == __version__
         assert attr_dict["applied_overrides"] == "foo"
         assert attr_dict["scenario"] == "bar"
+
+
+class TestTopLevelParams:
+    @pytest.fixture(scope="function")
+    def run_and_test(self):
+        def _run_and_test(in_dict, out_dict, dims):
+            model = build_model(
+                {"parameters.my_val": in_dict},
+                "simple_supply,two_hours",
+            )
+            _data = pd.Series(out_dict).rename_axis(index=dims)
+            pd.testing.assert_series_equal(
+                model.inputs.my_val.to_series().dropna().reindex(_data.index),
+                _data,
+                check_dtype=False,
+                check_names=False,
+                check_exact=False,
+            )
+
+        return _run_and_test
+
+    def test_protected_parameter_names(self):
+        with pytest.raises(KeyError) as excinfo:
+            build_model(
+                {"parameters.flow_eff.data": 1},
+                "simple_supply,two_hours",
+            )
+        assert check_error_or_warning(
+            excinfo,
+            "Trying to add top-level parameter with same name as a node/tech level parameter: flow_eff",
+        )
+
+    @pytest.mark.parametrize("val", [1, 1.0, np.inf, "foo"])
+    def test_top_level_param_single_val(self, val):
+        model = build_model(
+            {"parameters.my_val.data": val},
+            "simple_supply,two_hours",
+        )
+        assert model.inputs.my_val == xr.DataArray(val)
+
+    @pytest.mark.parametrize("val", [None, np.nan])
+    def test_top_level_param_single_val_cleaned_out_in_preprocessing(self, val):
+        model = build_model(
+            {"parameters.my_val.data": val},
+            "simple_supply,two_hours",
+        )
+        assert "my_val" not in model.inputs
+
+    def test_top_level_param_dims_no_index(self):
+        with pytest.raises(ValueError) as excinfo:
+            build_model(
+                {"parameters.my_val": {"data": 1, "dims": "techs"}},
+                "simple_supply,two_hours",
+            )
+        assert check_error_or_warning(
+            excinfo,
+            "(parameters, my_val) | Expected list for `index`, received: None",
+        )
+
+    def test_top_level_param_dims_not_list_index(self):
+        with pytest.raises(ValueError) as excinfo:
+            build_model(
+                {"parameters.my_val": {"data": 1, "dims": "techs", "index": "foo"}},
+                "simple_supply,two_hours",
+            )
+        assert check_error_or_warning(
+            excinfo,
+            "(parameters, my_val) | Expected list for `index`, received: foo",
+        )
+
+    @pytest.mark.parametrize("val", [1, 1.0, np.inf, "foo"])
+    def test_top_level_param_single_data_single_known_dim(self, val, run_and_test):
+        run_and_test(
+            {
+                "data": val,
+                "index": ["test_supply_elec"],
+                "dims": "techs",
+            },
+            {"test_supply_elec": val},
+            "techs",
+        )
+
+    def test_top_level_param_multi_data_single_known_dim(self, run_and_test):
+        run_and_test(
+            {
+                "data": [1, "foo"],
+                "index": ["test_supply_elec", "test_demand_elec"],
+                "dims": "techs",
+            },
+            {"test_supply_elec": 1, "test_demand_elec": "foo"},
+            "techs",
+        )
+
+    def test_top_level_param_single_data_multi_known_dim(self, run_and_test):
+        run_and_test(
+            {
+                "data": 10,
+                "index": [["a", "test_supply_elec"], ["b", "test_demand_elec"]],
+                "dims": ["nodes", "techs"],
+            },
+            {("a", "test_supply_elec"): 10, ("b", "test_demand_elec"): 10},
+            ["nodes", "techs"],
+        )
+
+    def test_top_level_param_multi_data_multi_known_dim(self, run_and_test):
+        run_and_test(
+            {
+                "data": [10, 20],
+                "index": [["a", "test_supply_elec"], ["b", "test_demand_elec"]],
+                "dims": ["nodes", "techs"],
+            },
+            {("a", "test_supply_elec"): 10, ("b", "test_demand_elec"): 20},
+            ["nodes", "techs"],
+        )
+
+    def test_top_level_param_unknown_dim_only(self, caplog, run_and_test):
+        caplog.set_level(logging.DEBUG)
+        run_and_test(
+            {"data": 10, "index": ["foo"], "dims": "bar"},
+            {"foo": 10},
+            "bar",
+        )
+        assert (
+            "(parameters, my_val) | Adding a new dimension to the model: bar"
+            in caplog.text
+        )
+
+    def test_top_level_param_multi_unknown_dim(self, caplog, run_and_test):
+        caplog.set_level(logging.DEBUG)
+        run_and_test(
+            {
+                "data": 10,
+                "index": [["foo", "foobar"]],
+                "dims": ["bar", "baz"],
+            },
+            {("foo", "foobar"): 10},
+            ["bar", "baz"],
+        )
+        assert (
+            "(parameters, my_val) | Adding a new dimension to the model: bar"
+            in caplog.text
+        )
+        assert (
+            "(parameters, my_val) | Adding a new dimension to the model: baz"
+            in caplog.text
+        )
+
+    def test_top_level_param_unknown_dim_mixed(self, caplog, run_and_test):
+        caplog.set_level(logging.DEBUG)
+        run_and_test(
+            {
+                "data": 10,
+                "index": [["test_supply_elec", "foobar"]],
+                "dims": ["techs", "baz"],
+            },
+            {("test_supply_elec", "foobar"): 10},
+            ["techs", "baz"],
+        )
+        assert (
+            "(parameters, my_val) | Adding a new dimension to the model: baz"
+            in caplog.text
+        )
+
+    def test_top_level_param_timeseries(self, caplog, run_and_test):
+        caplog.set_level(logging.DEBUG)
+        run_and_test(
+            {
+                "data": 10,
+                "index": ["2005-01-01"],
+                "dims": ["timesteps"],
+            },
+            {pd.to_datetime("2005-01-01"): 10},
+            "timesteps",
+        )
+        assert (
+            "(parameters, my_val) | Updating timesteps dimension index values to datetime format"
+            in caplog.text
+        )
+
+    @pytest.mark.filterwarnings(
+        "ignore:(?s).*Operational mode requires the same timestep resolution:calliope.exceptions.ModelWarning"
+    )
+    def test_top_level_param_extend_dim_vals(self, caplog, run_and_test):
+        # We do this test with timesteps as all other dimension elements are filtered out if there is no matching True element in `definition_matrix`
+        caplog.set_level(logging.DEBUG)
+        run_and_test(
+            {
+                "data": 10,
+                "index": ["2006-01-01"],
+                "dims": ["timesteps"],
+            },
+            {pd.to_datetime("2006-01-01"): 10},
+            "timesteps",
+        )
+        assert (
+            "(parameters, my_val) | Adding a new value to the `timesteps` model coordinate: ['2006-01-01T00:00:00.000000000']"
+            in caplog.text
+        )