fix: move all YAML operations to ruamel (#51)

pyproject.toml

@@ -12,7 +12,6 @@ license = { file = "LICENSE.txt" }
 dependencies = [
     "typing-extensions>=4.12,<5",
     "jinja2>=3.0.2,<4",
-    "types-PyYAML>=6.0.12.20240311,<6.0.13",
     "tomli>=2.0.1,<3",
     "ruamel.yaml>=0.18.6,<0.19",
 ]

src/rattler_build_conda_compat/jinja/jinja.py

@@ -3,7 +3,6 @@
 from typing import Any, TypedDict
 
 import jinja2
-import yaml
 from jinja2.sandbox import SandboxedEnvironment
 
 from rattler_build_conda_compat.jinja.filters import _bool, _split, _version_to_build_string
@@ -18,6 +17,7 @@
 )
 from rattler_build_conda_compat.jinja.utils import _MissingUndefined
 from rattler_build_conda_compat.loader import load_yaml
+from rattler_build_conda_compat.yaml import _dump_yaml_to_string
 
 
 class RecipeWithContext(TypedDict, total=False):
@@ -113,6 +113,7 @@ def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, A
 
     # render the rest of the document with the values from the context
     # and keep undefined expressions _as is_.
-    template = env.from_string(yaml.dump(recipe_content))
+    template = env.from_string(_dump_yaml_to_string(recipe_content))
     rendered_content = template.render(context_variables)
+
     return load_yaml(rendered_content)

src/rattler_build_conda_compat/lint.py

@@ -38,20 +38,9 @@ def get_recipe_schema() -> Dict[Any, Any]:
     return requests.get(SCHEMA_URL).json()
 
 
-def yaml_reader():
-    # define global yaml API
-    # roundrip-loader and allowing duplicate keys
-    # for handling # [filter] / # [not filter]
-    # Don't use a global variable for this as a global
-    # variable will make conda-smithy thread unsafe.
-    yaml = ruamel.yaml.YAML(typ="rt")
-    yaml.allow_duplicate_keys = True
-    return yaml
-
-
 def lint_recipe_yaml_by_schema(recipe_file):
     schema = get_recipe_schema()
-    yaml = yaml_reader()
+    yaml = _yaml_object()
 
     with open(recipe_file) as fh:
         meta = yaml.load(fh)

src/rattler_build_conda_compat/loader.py

@@ -1,15 +1,14 @@
 from __future__ import annotations
 
+import io
 import itertools
-from contextlib import contextmanager
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-import yaml
-
 from rattler_build_conda_compat.conditional_list import visit_conditional_list
+from rattler_build_conda_compat.yaml import _yaml_object
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
     from os import PathLike
 
 SELECTOR_OPERATORS = ("and", "or", "not")
@@ -38,104 +37,68 @@ def _flatten_lists(some_dict: dict[str, Any]) -> dict[str, Any]:
     return result_dict
 
 
-class RecipeLoader(yaml.BaseLoader):
-    _namespace: dict[str, Any] | None = None
-    _allow_missing_selector: bool = False
-
-    @classmethod
-    @contextmanager
-    def with_namespace(
-        cls: type[RecipeLoader],
-        namespace: dict[str, Any] | None,
-        *,
-        allow_missing_selector: bool = False,
-    ) -> Iterator[None]:
-        try:
-            cls._namespace = namespace
-            cls._allow_missing_selector = allow_missing_selector
-            yield
-        finally:
-            del cls._namespace
-
-    def construct_sequence(  # noqa: C901, PLR0912
-        self,
-        node: yaml.ScalarNode | yaml.SequenceNode | yaml.MappingNode,
-        deep: bool = False,  # noqa: FBT002, FBT001,
-    ) -> list[yaml.ScalarNode]:
-        """deep is True when creating an object/mapping recursively,
-        in that case want the underlying elements available during construction
-        """
-        # find if then else selectors
-        for sequence_idx, child_node in enumerate(node.value[:]):
-            # if then is only present in MappingNode
-
-            if isinstance(child_node, yaml.MappingNode):
-                # iterate to find if there is IF first
-
-                the_evaluated_one = None
-                for idx, (key_node, value_node) in enumerate(child_node.value):
-                    if key_node.value == "if":
-                        # we catch the first one, let's try to find next pair of (then | else)
-                        then_node_key, then_node_value = child_node.value[idx + 1]
-
-                        if then_node_key.value != "then":
-                            msg = "cannot have if without then, please reformat your variant file"
-                            raise ValueError(msg)
-
-                        try:
-                            _, else_node_value = child_node.value[idx + 2]
-                        except IndexError:
-                            _, else_node_value = None, None
-
-                        to_be_eval = f"{value_node.value}"
-
-                        if self._allow_missing_selector:
-                            split_selectors = [
-                                selector
-                                for selector in to_be_eval.split()
-                                if selector not in SELECTOR_OPERATORS
-                            ]
-                            for selector in split_selectors:
-                                if self._namespace and selector not in self._namespace:
-                                    cleaned_selector = selector.strip("(").rstrip(")")
-                                    self._namespace[cleaned_selector] = True
-
-                        evaled = eval(to_be_eval, self._namespace)  # noqa: S307
-                        if evaled:
-                            the_evaluated_one = then_node_value
-                        elif else_node_value:
-                            the_evaluated_one = else_node_value
-
-                        if the_evaluated_one:
-                            node.value.remove(child_node)
-                            node.value.insert(sequence_idx, the_evaluated_one)
-                        else:
-                            # neither the evaluation or else node is present, so we remove this if
-                            node.value.remove(child_node)
-
-        if not isinstance(node, yaml.SequenceNode):
-            raise TypeError(
-                None,
-                None,
-                f"expected a sequence node, but found {node.id!s}",
-                node.start_mark,
+def load_yaml(content: str) -> Any:  # noqa: ANN401
+    yaml = _yaml_object()
+    with io.StringIO(content) as f:
+        return yaml.load(f)
+
+
+def _eval_selector(
+    condition: str, namespace: dict[str, Any], *, allow_missing_selector: bool = False
+) -> bool:
+    # evaluate the selector expression
+    if allow_missing_selector:
+        namespace = namespace.copy()
+        split_selectors = [
+            selector for selector in condition.split() if selector not in SELECTOR_OPERATORS
+        ]
+        for selector in split_selectors:
+            if namespace and selector not in namespace:
+                cleaned_selector = selector.strip("(").rstrip(")")
+                namespace[cleaned_selector] = True
+
+    return eval(condition, namespace)  # noqa: S307
+
+
+def _render_recipe(
+    yaml_object: Any,  # noqa: ANN401
+    context: dict[str, Any],
+    *,
+    allow_missing_selector: bool = False,
+) -> Any:  # noqa: ANN401
+    # recursively go through the yaml object, and convert any lists with conditional if/else statements
+    # into a single list
+    if isinstance(yaml_object, dict):
+        for key, value in yaml_object.items():
+            yaml_object[key] = _render_recipe(
+                value, context, allow_missing_selector=allow_missing_selector
             )
-
-        return [self.construct_object(child, deep=deep) for child in node.value]
-
-
-def load_yaml(content: str | bytes) -> Any:  # noqa: ANN401
-    return yaml.load(content, Loader=yaml.BaseLoader)  # noqa: S506
+    elif isinstance(yaml_object, list):
+        # if the list is a conditional list, evaluate it
+        yaml_object = list(
+            visit_conditional_list(
+                yaml_object,
+                lambda x: _eval_selector(x, context, allow_missing_selector=allow_missing_selector),
+            )
+        )
+    return yaml_object
 
 
 def parse_recipe_config_file(
     path: PathLike[str], namespace: dict[str, Any] | None, *, allow_missing_selector: bool = False
 ) -> dict[str, Any]:
-    with open(path) as f, RecipeLoader.with_namespace(
-        namespace, allow_missing_selector=allow_missing_selector
-    ):
-        content = yaml.load(f, Loader=RecipeLoader)  # noqa: S506
-    return _flatten_lists(_remove_empty_keys(content))
+    yaml = _yaml_object()
+    with Path(path).open() as f:
+        raw_yaml_content = yaml.load(f)
+
+    # render the recipe with the context
+    if namespace is None:
+        namespace = {}
+
+    rendered = _render_recipe(
+        raw_yaml_content, namespace, allow_missing_selector=allow_missing_selector
+    )
+    return _flatten_lists(_remove_empty_keys(rendered))
 
 
 def load_all_requirements(content: dict[str, Any]) -> dict[str, Any]:

src/rattler_build_conda_compat/modify_recipe.py

@@ -2,16 +2,15 @@
 
 import copy
 import hashlib
-import io
 import logging
 import re
 from typing import TYPE_CHECKING, Any, Literal
 
 import requests
-from ruamel.yaml import YAML
 
 from rattler_build_conda_compat.jinja.jinja import jinja_env, load_recipe_context
 from rattler_build_conda_compat.recipe_sources import Source, get_all_sources
+from rattler_build_conda_compat.yaml import _dump_yaml_to_string, _yaml_object
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -20,11 +19,6 @@
 
 HashType = Literal["md5", "sha256"]
 
-yaml = YAML()
-yaml.preserve_quotes = True
-yaml.width = 4096
-yaml.indent(mapping=2, sequence=4, offset=2)
-
 
 def _update_build_number_in_context(recipe: dict[str, Any], new_build_number: int) -> bool:
     for key in recipe.get("context", {}):
@@ -62,15 +56,14 @@ def update_build_number(file: Path, new_build_number: int = 0) -> str:
     --------
     The updated recipe as a string.
     """
+    yaml = _yaml_object()
     with file.open("r") as f:
         data = yaml.load(f)
     build_number_modified = _update_build_number_in_context(data, new_build_number)
     if not build_number_modified:
         _update_build_number_in_recipe(data, new_build_number)
 
-    with io.StringIO() as f:
-        yaml.dump(data, f)
-        return f.getvalue()
+    return _dump_yaml_to_string(data)
 
 
 class CouldNotUpdateVersionError(Exception):
@@ -141,6 +134,7 @@ def update_version(file: Path, new_version: str, hash_: Hash | None) -> str:
     The updated recipe as a string.
     """
 
+    yaml = _yaml_object()
     with file.open("r") as f:
         data = yaml.load(f)
 
@@ -175,6 +169,4 @@ def update_version(file: Path, new_version: str, hash_: Hash | None) -> str:
 
         update_hash(source, rendered_url, hash_)
 
-    with io.StringIO() as f:
-        yaml.dump(data, f)
-        return f.getvalue()
+    return _dump_yaml_to_string(data)

src/rattler_build_conda_compat/render.py

@@ -10,8 +10,7 @@
 import sys
 import tempfile
 from typing import Any, Dict, List, Optional
-import yaml
-from ruamel.yaml import YAML
+
 from conda_build.metadata import (
     MetaData as CondaMetaData,
     OPTIONALLY_ITERABLE_FIELDS,
@@ -29,6 +28,7 @@
 from rattler_build_conda_compat.jinja.jinja import render_recipe_with_context
 from rattler_build_conda_compat.loader import load_yaml, parse_recipe_config_file
 from rattler_build_conda_compat.utils import _get_recipe_metadata, find_recipe
+from rattler_build_conda_compat.yaml import _yaml_object
 
 
 class MetaData(CondaMetaData):
@@ -117,13 +117,13 @@ def version(self) -> str:
 
     def render_recipes(self, variants) -> List[Dict]:
         platform_and_arch = f"{self.config.platform}-{self.config.arch}"
-
+        yaml = _yaml_object()
         try:
             with tempfile.NamedTemporaryFile(mode="w+") as outfile:
                 with tempfile.NamedTemporaryFile(mode="w") as variants_file:
                     # dump variants in our variants that will be used to generate recipe
                     if variants:
-                        yaml.dump(variants, variants_file, default_flow_style=False)
+                        yaml.dump(variants, variants_file)
 
                     variants_path = variants_file.name
 

src/rattler_build_conda_compat/yaml.py

@@ -0,0 +1,26 @@
+import io
+from typing import Any
+
+from ruamel.yaml import YAML
+
+
+# Custom constructor for loading floats as strings
+def float_as_string_constructor(loader, node) -> str:  # noqa: ANN001
+    return loader.construct_scalar(node)
+
+
+def _yaml_object() -> YAML:
+    yaml = YAML(typ="rt")
+    yaml.Constructor.add_constructor("tag:yaml.org,2002:float", float_as_string_constructor)
+    yaml.allow_duplicate_keys = False
+    yaml.preserve_quotes = True
+    yaml.width = 320
+    yaml.indent(mapping=2, sequence=4, offset=2)
+    return yaml
+
+
+def _dump_yaml_to_string(data: Any) -> str:  # noqa: ANN401
+    yaml = _yaml_object()
+    with io.StringIO() as f:
+        yaml.dump(data, f)
+        return f.getvalue()