Add config obj replace schemas

docs/hooks/generate_readable_schema.py

@@ -14,12 +14,13 @@
 import jsonschema2md
 from mkdocs.structure.files import File
 
+from calliope.schemas import config_schema
 from calliope.util import schema
 
 TEMPDIR = tempfile.TemporaryDirectory()
 
 SCHEMAS = {
-    "config_schema": schema.CONFIG_SCHEMA,
+    "config_schema": config_schema.CalliopeConfig().model_no_ref_schema(),
     "model_schema": schema.MODEL_SCHEMA,
     "math_schema": schema.MATH_SCHEMA,
     "data_table_schema": schema.DATA_TABLE_SCHEMA,

pyproject.toml

@@ -54,7 +54,7 @@ max-complexity = 10
 
 # Ignore `E402` (import violations) and `F401` (unused imports) in all `__init__.py` files
 [tool.ruff.lint.per-file-ignores]
-"__init__.py" = ["E402", "F401"]
+"__init__.py" = ["E402", "F401", "D104"]
 "*.ipynb" = ["E402"]
 "tests/*" = ["D"]
 "docs/examples/*" = ["D"]

requirements/base.txt

@@ -4,6 +4,7 @@ geographiclib >= 2, < 3
 ipdb >= 0.13, < 0.14
 ipykernel < 7
 jinja2 >= 3, < 4
+jsonref >= 1.1, < 2
 jsonschema >= 4, < 5
 natsort >= 8, < 9
 netcdf4 >= 1.2, < 1.7
@@ -13,4 +14,5 @@ pyomo >= 6.5, < 6.7.2
 pyparsing >= 3.0, < 3.1
 ruamel.yaml >= 0.18, < 0.19
 typing-extensions >= 4, < 5
-xarray >= 2024.1, < 2024.4
+xarray >= 2024.1, < 2024.4
+pydantic >= 2.9.2

src/calliope/attrdict.py

@@ -9,6 +9,7 @@
 
 import numpy as np
 import ruamel.yaml as ruamel_yaml
+from ruamel.yaml.scalarstring import walk_tree
 from typing_extensions import Self
 
 from calliope.util.tools import relative_path
@@ -328,12 +329,8 @@ def as_dict_flat(self):
             d[k] = self.get_key(k)
         return d
 
-    def to_yaml(self, path=None):
-        """Conversion to YAML.
-
-        Saves the AttrDict to the ``path`` as a YAML file or returns a YAML string
-        if ``path`` is None.
-        """
+    def to_yaml(self, path: str | None = None) -> str:
+        """Return a serialised YAML string."""
         result = self.copy()
         yaml_ = ruamel_yaml.YAML()
         yaml_.indent = 2
@@ -355,13 +352,16 @@ def to_yaml(self, path=None):
 
         result = result.as_dict()
 
-        if path is not None:
+        # handle multi-line strings.
+        walk_tree(result)
+
+        stream = io.StringIO()
+        yaml_.dump(result, stream)
+        yaml_str = stream.getvalue()
+        if path:
             with open(path, "w") as f:
-                yaml_.dump(result, f)
-        else:
-            stream = io.StringIO()
-            yaml_.dump(result, stream)
-            return stream.getvalue()
+                f.write(yaml_str)
+        return yaml_str
 
     def keys_nested(self, subkeys_as="list"):
         """Returns all keys in the AttrDict, including nested keys.

src/calliope/backend/__init__.py

@@ -16,29 +16,29 @@
 
 if TYPE_CHECKING:
     from calliope.backend.backend_model import BackendModel
+    from calliope.schemas import config_schema
 
 
 def get_model_backend(
-    name: str, data: xr.Dataset, math: CalliopeMath, **kwargs
+    build_config: "config_schema.Build", data: xr.Dataset, math: CalliopeMath
 ) -> "BackendModel":
     """Assign a backend using the given configuration.
 
     Args:
-        name (str): name of the backend to use.
+        build_config: Build configuration options.
         data (Dataset): model data for the backend.
         math (CalliopeMath): Calliope math.
-        **kwargs: backend keyword arguments corresponding to model.config.build.
 
     Raises:
         exceptions.BackendError: If invalid backend was requested.
 
     Returns:
         BackendModel: Initialized backend object.
     """
-    match name:
+    match build_config.backend:
         case "pyomo":
-            return PyomoBackendModel(data, math, **kwargs)
+            return PyomoBackendModel(data, math, build_config)
         case "gurobi":
-            return GurobiBackendModel(data, math, **kwargs)
+            return GurobiBackendModel(data, math, build_config)
         case _:
-            raise BackendError(f"Incorrect backend '{name}' requested.")
+            raise BackendError(f"Incorrect backend '{build_config.backend}' requested.")

src/calliope/backend/backend_model.py

@@ -32,11 +32,8 @@
 from calliope.exceptions import warn as model_warn
 from calliope.io import load_config
 from calliope.preprocess.model_math import ORDERED_COMPONENTS_T, CalliopeMath
-from calliope.util.schema import (
-    MODEL_SCHEMA,
-    extract_from_schema,
-    update_then_validate_config,
-)
+from calliope.schemas import config_schema
+from calliope.util.schema import MODEL_SCHEMA, extract_from_schema
 
 if TYPE_CHECKING:
     from calliope.backend.parsing import T as Tp
@@ -69,20 +66,20 @@ class BackendModelGenerator(ABC):
     _PARAM_UNITS = extract_from_schema(MODEL_SCHEMA, "x-unit")
     _PARAM_TYPE = extract_from_schema(MODEL_SCHEMA, "x-type")
 
-    def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs):
+    def __init__(
+        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build
+    ):
         """Abstract base class to build a representation of the optimisation problem.
 
         Args:
             inputs (xr.Dataset): Calliope model data.
             math (CalliopeMath): Calliope math.
-            **kwargs (Any): build configuration overrides.
+            build_config: Build configuration options.
         """
         self._dataset = xr.Dataset()
         self.inputs = inputs.copy()
         self.inputs.attrs = deepcopy(inputs.attrs)
-        self.inputs.attrs["config"]["build"] = update_then_validate_config(
-            "build", self.inputs.attrs["config"], **kwargs
-        )
+        self.config = build_config
         self.math: CalliopeMath = deepcopy(math)
         self._solve_logger = logging.getLogger(__name__ + ".<solve>")
 
@@ -200,6 +197,7 @@ def _check_inputs(self):
             "equation_name": "",
             "backend_interface": self,
             "input_data": self.inputs,
+            "build_config": self.config,
             "helper_functions": helper_functions._registry["where"],
             "apply_where": True,
             "references": set(),
@@ -246,7 +244,7 @@ def add_optimisation_components(self) -> None:
         # The order of adding components matters!
         # 1. Variables, 2. Global Expressions, 3. Constraints, 4. Objectives
         self._add_all_inputs_as_parameters()
-        if self.inputs.attrs["config"]["build"]["pre_validate_math_strings"]:
+        if self.config.pre_validate_math_strings:
             self._validate_math_string_parsing()
         for components in typing.get_args(ORDERED_COMPONENTS_T):
             component = components.removesuffix("s")
@@ -399,7 +397,7 @@ def _add_all_inputs_as_parameters(self) -> None:
             if param_name in self.parameters.keys():
                 continue
             elif (
-                self.inputs.attrs["config"]["build"]["mode"] != "operate"
+                self.config.mode != "operate"
                 and param_name
                 in extract_from_schema(MODEL_SCHEMA, "x-operate-param").keys()
             ):
@@ -606,17 +604,21 @@ class BackendModel(BackendModelGenerator, Generic[T]):
     """Calliope's backend model functionality."""
 
     def __init__(
-        self, inputs: xr.Dataset, math: CalliopeMath, instance: T, **kwargs
+        self,
+        inputs: xr.Dataset,
+        math: CalliopeMath,
+        instance: T,
+        build_config: config_schema.Build,
     ) -> None:
         """Abstract base class to build backend models that interface with solvers.
 
         Args:
             inputs (xr.Dataset): Calliope model data.
             math (CalliopeMath): Calliope math.
             instance (T): Interface model instance.
-            **kwargs: build configuration overrides.
+            build_config: Build configuration options.
         """
-        super().__init__(inputs, math, **kwargs)
+        super().__init__(inputs, math, build_config)
         self._instance = instance
         self.shadow_prices: ShadowPrices
         self._has_verbose_strings: bool = False

src/calliope/backend/gurobi_backend_model.py

@@ -18,6 +18,7 @@
 from calliope.exceptions import BackendError, BackendWarning
 from calliope.exceptions import warn as model_warn
 from calliope.preprocess import CalliopeMath
+from calliope.schemas import config_schema
 
 if importlib.util.find_spec("gurobipy") is not None:
     import gurobipy
@@ -41,19 +42,21 @@
 class GurobiBackendModel(backend_model.BackendModel):
     """gurobipy-specific backend functionality."""
 
-    def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs) -> None:
+    def __init__(
+        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build
+    ) -> None:
         """Gurobi solver interface class.
 
         Args:
             inputs (xr.Dataset): Calliope model data.
             math (CalliopeMath): Calliope math.
-            **kwargs: passed directly to the solver.
+            build_config: Build configuration options.
         """
         if importlib.util.find_spec("gurobipy") is None:
             raise ImportError(
                 "Install the `gurobipy` package to build the optimisation problem with the Gurobi backend."
             )
-        super().__init__(inputs, math, gurobipy.Model(), **kwargs)
+        super().__init__(inputs, math, gurobipy.Model(), build_config)
         self._instance: gurobipy.Model
         self.shadow_prices = GurobiShadowPrices(self)
 
@@ -144,7 +147,7 @@ def _objective_setter(
         ) -> xr.DataArray:
             expr = element.evaluate_expression(self, references=references)
 
-            if name == self.inputs.attrs["config"].build.objective:
+            if name == self.config.objective:
                 self._instance.setObjective(expr.item(), sense=sense)
 
                 self.log("objectives", name, "Objective activated.")

src/calliope/backend/parsing.py

@@ -311,6 +311,7 @@ def evaluate_where(
                 helper_functions=helper_functions._registry["where"],
                 input_data=backend_interface.inputs,
                 backend_interface=backend_interface,
+                build_config=backend_interface.config,
                 references=references if references is not None else set(),
                 apply_where=True,
             )

src/calliope/backend/pyomo_backend_model.py

@@ -29,6 +29,7 @@
 from calliope.exceptions import BackendError, BackendWarning
 from calliope.exceptions import warn as model_warn
 from calliope.preprocess import CalliopeMath
+from calliope.schemas import config_schema
 from calliope.util.logging import LogWriter
 
 from . import backend_model, parsing
@@ -58,15 +59,17 @@
 class PyomoBackendModel(backend_model.BackendModel):
     """Pyomo-specific backend functionality."""
 
-    def __init__(self, inputs: xr.Dataset, math: CalliopeMath, **kwargs) -> None:
+    def __init__(
+        self, inputs: xr.Dataset, math: CalliopeMath, build_config: config_schema.Build
+    ) -> None:
         """Pyomo solver interface class.
 
         Args:
             inputs (xr.Dataset): Calliope model data.
             math (CalliopeMath): Calliope math.
-            **kwargs: passed directly to the solver.
+            build_config: Build configuration options.
         """
-        super().__init__(inputs, math, pmo.block(), **kwargs)
+        super().__init__(inputs, math, pmo.block(), build_config)
 
         self._instance.parameters = pmo.parameter_dict()
         self._instance.variables = pmo.variable_dict()
@@ -185,7 +188,7 @@ def _objective_setter(
         ) -> xr.DataArray:
             expr = element.evaluate_expression(self, references=references)
             objective = pmo.objective(expr.item(), sense=sense)
-            if name == self.inputs.attrs["config"].build.objective:
+            if name == self.config.objective:
                 text = "activated"
                 objective.activate()
             else:

src/calliope/backend/where_parser.py

@@ -15,13 +15,13 @@
 
 from calliope.backend import expression_parser
 from calliope.exceptions import BackendError
+from calliope.schemas import config_schema
 
 if TYPE_CHECKING:
     from calliope.backend.backend_model import BackendModel
 
 
 pp.ParserElement.enablePackrat()
-
 BOOLEANTYPE = np.bool_ | np.typing.NDArray[np.bool_]
 
 
@@ -34,6 +34,7 @@ class EvalAttrs(TypedDict):
     helper_functions: dict[str, Callable]
     apply_where: NotRequired[bool]
     references: NotRequired[set]
+    build_config: config_schema.Build
 
 
 class EvalWhere(expression_parser.EvalToArrayStr):
@@ -118,9 +119,7 @@ def as_math_string(self) -> str:  # noqa: D102, override
         return rf"\text{{config.{self.config_option}}}"
 
     def as_array(self) -> xr.DataArray:  # noqa: D102, override
-        config_val = (
-            self.eval_attrs["input_data"].attrs["config"].build[self.config_option]
-        )
+        config_val = getattr(self.eval_attrs["build_config"], self.config_option)
 
         if not isinstance(config_val, int | float | str | bool | np.bool_):
             raise BackendError(

src/calliope/cli.py

@@ -278,9 +278,9 @@ def run(
         # Else run the model, then save outputs
         else:
             click.secho("Starting model run...")
-
+            kwargs = {}
             if save_logs:
-                model.config.set_key("solve.save_logs", save_logs)
+                kwargs["solve.save_logs"] = save_logs
 
             if save_csv is None and save_netcdf is None:
                 click.secho(
@@ -292,14 +292,13 @@ def run(
             # If save_netcdf is used, override the 'save_per_spore_path' to point to a
             # directory of the same name as the planned netcdf
 
-            if save_netcdf and model.config.solve.spores_save_per_spore:
-                model.config.set_key(
-                    "solve.spores_save_per_spore_path",
+            if save_netcdf and model.config.solve.spores.save_per_spore:
+                kwargs["solve.spores_save_per_spore_path"] = (
                     save_netcdf.replace(".nc", "/spore_{}.nc"),
                 )
 
             model.build()
-            model.solve()
+            model.solve(**kwargs)
             termination = model._model_data.attrs.get(
                 "termination_condition", "unknown"
             )