Merge pull request #108 from nolar/recursive-labelling

Label sub-templates recursively

kopf/reactor/handling.py

@@ -26,6 +26,7 @@
 from kopf.reactor import causation
 from kopf.reactor import invocation
 from kopf.reactor import registries
+from kopf.structs import dicts
 from kopf.structs import diffs
 from kopf.structs import finalizers
 from kopf.structs import lastseen
@@ -445,8 +446,8 @@ async def _call_handler(
     """
 
     # For the field-handlers, the old/new/diff values must match the field, not the whole object.
-    old = cause.old if handler.field is None else diffs.resolve(cause.old, handler.field, None)
-    new = cause.new if handler.field is None else diffs.resolve(cause.new, handler.field, None)
+    old = cause.old if handler.field is None else dicts.resolve(cause.old, handler.field, None)
+    new = cause.new if handler.field is None else dicts.resolve(cause.new, handler.field, None)
     diff = cause.diff if handler.field is None else diffs.reduce(cause.diff, handler.field)
     cause = cause._replace(old=old, new=new, diff=diff)
 

kopf/structs/dicts.py

@@ -0,0 +1,70 @@
+"""
+Some basic dicts and field-in-a-dict manipulation helpers.
+"""
+import collections.abc
+from typing import Any, Union, Mapping, Tuple, List, Text, Iterable, Optional
+
+FieldPath = Tuple[str, ...]
+FieldSpec = Union[None, Text, FieldPath, List[str]]
+
+_UNSET = object()
+
+
+def parse_field(
+        field: FieldSpec,
+) -> FieldPath:
+    """
+    Convert any field into a tuple of nested sub-fields.
+    """
+    if field is None:
+        return tuple()
+    elif isinstance(field, str):
+        return tuple(field.split('.'))
+    elif isinstance(field, (list, tuple)):
+        return tuple(field)
+    else:
+        raise ValueError(f"Field must be either a str, or a list/tuple. Got {field!r}")
+
+
+def resolve(
+        d: Mapping,
+        field: FieldSpec,
+        default: Any = _UNSET,
+):
+    """
+    Retrieve a nested sub-field from a dict.
+    """
+    path = parse_field(field)
+    try:
+        result = d
+        for key in path:
+            result = result[key]
+        return result
+    except KeyError:
+        if default is _UNSET:
+            raise
+        else:
+            return default
+
+
+def walk(
+        objs,
+        nested: Optional[Iterable[FieldSpec]] = None,
+):
+    """
+    Iterate over one or many dicts (and sub-dicts recursively).
+    """
+    if objs is None:
+        return
+    elif isinstance(objs, collections.abc.Mapping):
+        yield objs
+        for subfield in (nested if nested is not None else []):
+            try:
+                yield resolve(objs, parse_field(subfield))
+            except KeyError:
+                pass
+    elif isinstance(objs, collections.abc.Iterable):
+        for obj in objs:
+            yield from walk(obj, nested=nested)
+    else:
+        yield objs  # NB: not a mapping, no nested sub-fields.

kopf/structs/diffs.py

@@ -1,31 +1,15 @@
 """
 All the functions to calculate the diffs of the dicts.
 """
-
 import collections
 
-from typing import Any, Tuple, NewType, Generator, Sequence, Mapping
+from typing import Any, Tuple, NewType, Generator, Sequence
 
 DiffOp = NewType('DiffOp', str)
 DiffPath = Tuple[str, ...]
 DiffItem = Tuple[DiffOp, DiffPath, Any, Any]
 Diff = Sequence[DiffItem]
 
-_UNSET = object()
-
-
-def resolve(d: Mapping, path: DiffPath, default=_UNSET):
-    try:
-        result = d
-        for key in path:
-            result = result[key]
-        return result
-    except KeyError:
-        if default is _UNSET:
-            raise
-        else:
-            return default
-
 
 def reduce_iter(d: Diff, path: DiffPath) -> Generator[DiffItem, None, None]:
     for op, field, old, new in d:

kopf/structs/hierarchies.py

@@ -1,7 +1,7 @@
 """
 All the functions to properly build the object hierarchies.
 """
-import collections.abc
+from kopf.structs import dicts
 
 
 def build_object_reference(body):
@@ -42,7 +42,7 @@ def append_owner_reference(objs, owner):
     so the whole body can be modified, no patches are needed.
     """
     owner = build_owner_reference(owner)
-    for obj in _iter_objects(objs):
+    for obj in dicts.walk(objs):
         refs = obj.setdefault('metadata', {}).setdefault('ownerReferences', [])
         matching = [ref for ref in refs if ref['uid'] == owner['uid']]
         if not matching:
@@ -57,19 +57,18 @@ def remove_owner_reference(objs, owner):
     so the whole body can be modified, no patches are needed.
     """
     owner = build_owner_reference(owner)
-    for obj in _iter_objects(objs):
+    for obj in dicts.walk(objs):
         refs = obj.setdefault('metadata', {}).setdefault('ownerReferences', [])
         matching = [ref for ref in refs if ref['uid'] == owner['uid']]
         for ref in matching:
             refs.remove(ref)
 
 
-# TODO: make it also recursively if there are any .metadata.labels inside (e.g. job/pod templates).
-def label(objs, labels, force=False):
+def label(objs, labels, *, force=False, nested=None):
     """
     Apply the labels to the object(s).
     """
-    for obj in _iter_objects(objs):
+    for obj in dicts.walk(objs, nested=nested):
         obj_labels = obj.setdefault('metadata', {}).setdefault('labels', {})
         for key, val in labels.items():
             if force:
@@ -93,7 +92,7 @@ def harmonize_naming(objs, name=None, strict=False):
     If the objects already have their own names, auto-naming is not applied,
     and the existing names are used as is.
     """
-    for obj in _iter_objects(objs):
+    for obj in dicts.walk(objs):
         if obj.get('metadata', {}).get('name', None) is None:
             if strict:
                 obj.setdefault('metadata', {}).setdefault('name', name)
@@ -110,29 +109,15 @@ def adjust_namespace(objs, namespace=None):
     It is a common practice to keep the children objects in the same
     namespace as their owner, unless explicitly overridden at time of creation.
     """
-    for obj in _iter_objects(objs):
+    for obj in dicts.walk(objs):
         obj.setdefault('metadata', {}).setdefault('namespace', namespace)
 
 
-def adopt(objs, owner):
+def adopt(objs, owner, *, nested=None):
     """
     The children should be in the same namespace, named after their parent, and owned by it.
     """
     append_owner_reference(objs, owner=owner)
     harmonize_naming(objs, name=owner.get('metadata', {}).get('name', None))
     adjust_namespace(objs, namespace=owner.get('metadata', {}).get('namespace', None))
-    label(objs, labels=owner.get('metadata', {}).get('labels', {}))
-
-
-def _iter_objects(objs):
-    """
-    A helper to iterate over one or many objects, as used in these functions.
-    """
-    if objs is None:
-        return
-    elif isinstance(objs, collections.abc.Mapping):  # also iterable, intercept it.
-        yield objs
-    elif isinstance(objs, collections.abc.Iterable):
-        yield from objs
-    else:
-        yield objs
+    label(objs, labels=owner.get('metadata', {}).get('labels', {}), nested=nested)

tests/diffs-n-dicts/test_parsing.py

@@ -0,0 +1,39 @@
+import pytest
+
+from kopf.structs.dicts import parse_field
+
+
+def test_from_none():
+    path = parse_field(None)
+    assert isinstance(path, tuple)
+    assert len(path) == 0
+
+
+def test_from_string_one_level():
+    path = parse_field('field')
+    assert isinstance(path, tuple)
+    assert path == ('field',)
+
+
+def test_from_string_two_levels():
+    path = parse_field('field.subfield')
+    assert isinstance(path, tuple)
+    assert path == ('field', 'subfield')
+
+
+def test_from_list():
+    path = parse_field(['field' , 'subfield'])
+    assert isinstance(path, tuple)
+    assert path == ('field', 'subfield')
+
+
+def test_from_tuple():
+    path = parse_field(('field' , 'subfield'))
+    assert isinstance(path, tuple)
+    assert path == ('field', 'subfield')
+
+
+@pytest.mark.parametrize('val', [dict(), set(), frozenset()])
+def test_from_others_fails(val):
+    with pytest.raises(ValueError):
+        parse_field(val)

tests/diffs/test_resolving.py → tests/diffs-n-dicts/test_resolving.py

@@ -1,6 +1,6 @@
 import pytest
 
-from kopf.structs.diffs import resolve
+from kopf.structs.dicts import resolve
 
 
 def test_existing_key():

tests/diffs-n-dicts/test_walking.py

@@ -0,0 +1,62 @@
+from kopf.structs.dicts import walk
+
+
+def test_over_a_none():
+    result = list(walk(None))
+    assert len(result) == 0
+
+
+def test_over_a_dict():
+    obj = {}
+    result = list(walk(obj))
+    assert len(result) == 1
+    assert result[0] is obj
+
+
+def test_over_a_list_of_dicts():
+    obj1 = {}
+    obj2 = {}
+    result = list(walk([obj1, obj2]))
+    assert len(result) == 2
+    assert result[0] is obj1
+    assert result[1] is obj2
+
+
+def test_over_a_tuple_of_dicts():
+    obj1 = {}
+    obj2 = {}
+    result = list(walk((obj1, obj2)))
+    assert len(result) == 2
+    assert result[0] is obj1
+    assert result[1] is obj2
+
+
+def test_none_is_ignored():
+    obj1 = {}
+    obj2 = {}
+    result = list(walk([obj1, None, obj2]))
+    assert len(result) == 2
+    assert result[0] is obj1
+    assert result[1] is obj2
+
+
+def test_simple_nested():
+    obj1 = {'field': {'subfield': 'val'}}
+    obj2 = {'field': {}}
+    result = list(walk([obj1, obj2], nested=['field.subfield']))
+    assert len(result) == 3
+    assert result[0] is obj1
+    assert result[1] == 'val'
+    assert result[2] is obj2
+
+
+def test_double_nested():
+    obj1 = {'field': {'subfield': 'val'}}
+    obj2 = {'field': {}}
+    result = list(walk([obj1, obj2], nested=['field.subfield', 'field']))
+    assert len(result) == 5
+    assert result[0] is obj1
+    assert result[1] == 'val'
+    assert result[2] == {'subfield': 'val'}
+    assert result[3] is obj2
+    assert result[4] == {}

tests/hierarchies/test_labelling.py

@@ -54,3 +54,19 @@ def test_forcing_default():
     obj = {'metadata': {'labels': {'label': 'old-value'}}}
     kopf.label(obj, {'label': 'new-value'})
     assert obj['metadata']['labels']['label'] == 'old-value'
+
+
+def test_nested_with_forced_true():
+    obj = {'metadata': {'labels': {'label': 'old-value'}},
+           'spec': {'template': {}}}
+    kopf.label(obj, {'label': 'new-value'}, nested=['spec.template'], force=True)
+    assert obj['metadata']['labels']['label'] == 'new-value'
+    assert obj['spec']['template']['metadata']['labels']['label'] == 'new-value'
+
+
+def test_nested_with_forced_false():
+    obj = {'metadata': {'labels': {'label': 'old-value'}},
+           'spec': {'template': {}}}
+    kopf.label(obj, {'label': 'new-value'}, nested=['spec.template'], force=False)
+    assert obj['metadata']['labels']['label'] == 'old-value'
+    assert obj['spec']['template']['metadata']['labels']['label'] == 'new-value'

tests/hierarchies/test_owner_referencing.py

@@ -193,7 +193,7 @@ def test_adopting(mocker):
     label = mocker.patch('kopf.structs.hierarchies.label')
 
     obj = Mock()
-    kopf.adopt(obj, owner=OWNER)
+    kopf.adopt(obj, owner=OWNER, nested=['template'])
 
     assert append_owner_ref.called
     assert harmonize_naming.called
@@ -203,4 +203,4 @@ def test_adopting(mocker):
     assert append_owner_ref.call_args_list == [call(obj, owner=OWNER)]
     assert harmonize_naming.call_args_list == [call(obj, name=OWNER_NAME)]
     assert adjust_namespace.call_args_list == [call(obj, namespace=OWNER_NAMESPACE)]
-    assert label.call_args_list == [call(obj, labels=OWNER_LABELS)]
+    assert label.call_args_list == [call(obj, labels=OWNER_LABELS, nested=['template'])]