WIP2

karapace/protobuf/compare_type_lists.py

@@ -19,6 +19,7 @@ def compare_type_lists(
     other_types_list: Sequence[TypeElement],
     result: CompareResult,
     compare_types: CompareTypes,
+    compare_full_path: bool = False,
 ) -> CompareResult:
     self_types = {}
     other_types = {}

karapace/protobuf/enum_element.py

@@ -61,7 +61,7 @@ def to_schema(self) -> str:
         result.append("}\n")
         return "".join(result)
 
-    def compare(self, other: TypeElement, result: CompareResult, types: CompareTypes) -> None:
+    def compare(self, other: TypeElement, result: CompareResult, types: CompareTypes, compare_full_path: bool = False) -> None:
         self_tags = {}
         other_tags = {}
         if types:

karapace/protobuf/extend_element.py

@@ -23,7 +23,7 @@ class ExtendElement:
 
     def with_full_path_expanded(self, type_tree: TypeTree) -> ExtendElement:
         # read again carefully there -> https://protobuf.com/docs/language-spec#fully-qualified-references
-        full_path_fields = [field.with_full_path_expanded(type_tree) for field in self.fields]
+        full_path_fields = [field.with_full_path_expanded(type_tree) for field in self.fields] if self.fields else None
         return ExtendElement(
             location=self.location,
             name=self.name,

karapace/protobuf/field_element.py

@@ -47,6 +47,13 @@ def __init__(
 
     def with_full_path_expanded(self, type_tree: TypeTree) -> FieldElement:
         full_path_element_type = self.element_type
+        element_type_tokens = self.element_type.split(".")
+        maybe_path_in_tree = type_tree.type_in_tree(element_type_tokens)
+        if maybe_path_in_tree:
+            missing_tokens = maybe_path_in_tree.expand_missing_absolute_path()
+        else:
+            missing_tokens = []
+
         full_path_options = self.options
         return FieldElement(
             location=self.location,
@@ -58,6 +65,7 @@ def with_full_path_expanded(self, type_tree: TypeTree) -> FieldElement:
             tag=self.tag,
             documentation=self.documentation,
             options=full_path_options,
+            fully_qualified_name=".".join(missing_tokens + element_type_tokens)
         )
 
     def to_schema(self) -> str:

karapace/protobuf/group_element.py

@@ -25,7 +25,8 @@ class GroupElement:
     fields: Sequence[FieldElement] | None = None
 
     def with_full_path_expanded(self, type_tree: TypeTree) -> GroupElement:
-        full_path_fields = [field.with_full_path_expanded(type_tree) for field in self.fields]
+        full_path_fields = [field.with_full_path_expanded(type_tree) for field in self.fields] if self.fields else None
+        # here also
         return GroupElement(
             label=self.label,
             location=self.location,

karapace/protobuf/message_element.py

@@ -28,8 +28,17 @@ class MessageElement(TypeElement):
     fields: Sequence[FieldElement]
     one_ofs: Sequence[OneOfElement]
     groups: Sequence[GroupElement]
+    fully_qualified_name: str | None = None
 
     def with_full_path_expanded(self, type_tree: TypeTree) -> MessageElement:
+        # here also
+
+        element_type_tokens = self.name.split(".")
+        maybe_path_in_tree = type_tree.type_in_tree(element_type_tokens)
+        if maybe_path_in_tree:
+            missing_tokens = maybe_path_in_tree.expand_missing_absolute_path()
+        else:
+            missing_tokens = []
         full_path_nested_types = [nested_type.with_full_path_expanded(type_tree) for nested_type in self.nested_types]
         full_path_options = [option.with_full_path_expanded(type_tree) for option in self.options]
         full_path_fields = [field.with_full_path_expanded(type_tree) for field in self.fields]
@@ -110,7 +119,7 @@ def to_schema(self) -> str:
         result.append("}\n")
         return "".join(result)
 
-    def compare(self, other: TypeElement, result: CompareResult, types: CompareTypes) -> None:
+    def compare(self, other: TypeElement, result: CompareResult, types: CompareTypes, compare_full_path: bool = False) -> None:
         from karapace.protobuf.compare_type_lists import compare_type_lists
 
         if not isinstance(other, MessageElement):

karapace/protobuf/option_element.py

@@ -13,7 +13,7 @@
 
 class OptionElement:
     name: str
-    fully_qualified_name: str
+    fully_qualified_name: str | None = None
 
     class Kind(Enum):
         STRING = 1
@@ -33,6 +33,12 @@ def __init__(self, name: str, kind: Kind, value, is_parenthesized: bool = False)
         self.formattedName = f"({self.name})" if is_parenthesized else self.name
 
     def with_full_path_expanded(self, type_tree: TypeTree) -> OptionElement:
+        # fully qualified name also here.
+
+        #full_path_element_type = self.element_type
+        #element_type_tokens = self.element_type.split(".")
+        #missing_tokens = type_tree.type_in_tree(element_type_tokens).expand_missing_absolute_path()
+
         return OptionElement(
             name=self.name,  # understand if the name should be changed, otherwise remove the method.
             kind=self.kind,
@@ -53,9 +59,9 @@ def to_schema(self) -> str:
         elif self.kind == self.Kind.OPTION:
             aline = f"{self.formattedName}.{try_to_schema(self.value)}"
         elif self.kind == self.Kind.MAP:
-            aline = [f"{self.formattedName} = {{\n", self.format_option_map(self.value), "}"]
+            aline = [f"{self.formattedName} = {{\n", OptionElement.format_option_map(self.value), "}"]
         elif self.kind == self.Kind.LIST:
-            aline = [f"{self.formattedName} = ", self.append_options(self.value)]
+            aline = [f"{self.formattedName} = ", OptionElement.append_options(self.value)]
         else:
             raise ValueError("Unknown value Kind.")
 
@@ -72,36 +78,41 @@ def append_options(options: list) -> str:
         append_options(data, options)
         return "".join(data)
 
-    def format_option_map(self, value: dict) -> str:
+
+    @staticmethod
+    def format_option_map(value: dict) -> str:
         keys = list(value.keys())
         last_index = len(keys) - 1
         result: list[str] = []
         for index, key in enumerate(keys):
             endl = "," if (index != last_index) else ""
-            append_indented(result, f"{key}: {self.format_option_map_value(value[key])}{endl}")
+            append_indented(result, f"{key}: {OptionElement.format_option_map_value(value[key])}{endl}")
         return "".join(result)
 
-    def format_option_map_value(self, value) -> str:
+
+    @staticmethod
+    def format_option_map_value(value) -> str:
         aline = value
         if isinstance(value, str):
             aline = f'"{value}"'
         elif isinstance(value, dict):
-            aline = ["{\n", self.format_option_map(value), "}"]
+            aline = ["{\n", OptionElement.format_option_map(value), "}"]
         elif isinstance(value, list):
-            aline = ["[\n", self.format_list_map_value(value), "]"]
+            aline = ["[\n", OptionElement.format_list_map_value(value), "]"]
 
         if isinstance(aline, list):
             return "".join(aline)
         if isinstance(aline, str):
             return aline
         return value
 
-    def format_list_map_value(self, value) -> str:
+    @staticmethod
+    def format_list_map_value(value) -> str:
         last_index = len(value) - 1
         result: list[str] = []
         for index, elm in enumerate(value):
             endl = "," if (index != last_index) else ""
-            append_indented(result, f"{self.format_option_map_value(elm)}{endl}")
+            append_indented(result, f"{OptionElement.format_option_map_value(elm)}{endl}")
         return "".join(result)
 
     def __repr__(self) -> str:

karapace/protobuf/proto_file_element.py

@@ -155,6 +155,7 @@ def compare(
         result: CompareResult,
         self_dependencies: dict[str, Dependency] | None = None,
         other_dependencies: dict[str, Dependency] | None = None,
+        compare_full_path: bool = False
     ) -> CompareResult:
         from karapace.protobuf.compare_type_lists import compare_type_lists
 
@@ -170,4 +171,4 @@ def compare(
 
         if other_dependencies:
             _collect_dependencies_types(compare_types, other_dependencies, False)
-        return compare_type_lists(self.types, other.types, result, compare_types)
+        return compare_type_lists(self.types, other.types, result, compare_types, compare_full_path)

karapace/protobuf/proto_normalizations.py

@@ -14,7 +14,7 @@
 from karapace.protobuf.message_element import MessageElement
 from karapace.protobuf.one_of_element import OneOfElement
 from karapace.protobuf.option_element import OptionElement
-from karapace.protobuf.proto_file_element import ProtoFileElement
+from karapace.protobuf.proto_file_element import PackageName, ProtoFileElement
 from karapace.protobuf.rpc_element import RpcElement
 from karapace.protobuf.schema import ProtobufSchema, TypeTree
 from karapace.protobuf.service_element import ServiceElement
@@ -264,7 +264,7 @@ def normalize(proto_file_element: ProtoFileElement, type_tree: TypeTree) -> Norm
 
     return NormalizedProtoFileElement(
         location=full_path_proto_file_element.location,
-        package_name=full_path_proto_file_element.package_name,
+        package_name=PackageName(full_path_proto_file_element.package_name),
         syntax=full_path_proto_file_element.syntax,
         imports=full_path_proto_file_element.imports,
         public_imports=full_path_proto_file_element.public_imports,

karapace/protobuf/schema.py

@@ -147,7 +147,7 @@ def _add_new_type_recursive(
             source_reference=(
                 None if remaining_tokens else SourceFileReference(reference=file, import_order=inserted_elements)
             ),
-            type_provider=None if remaining_tokens else type_provider,
+            provider=None if remaining_tokens else type_provider,
         )
         parent_tree.children.append(new_leaf)
         return _add_new_type_recursive(new_leaf, remaining_tokens, file, inserted_elements, type_provider)
@@ -189,16 +189,6 @@ def __init__(
         self.references = references
         self.dependencies = dependencies
 
-    def type_in_tree(self, tree: TypeTree, remaining_tokens: list[str]) -> TypeTree | None:
-        if remaining_tokens:
-            to_seek = remaining_tokens.pop()
-
-            for child in tree.children:
-                if child.token == to_seek:
-                    return self.type_in_tree(child, remaining_tokens)
-            return None
-        return tree
-
     def record_name(self) -> str | None:
         if len(self.proto_file_element.types) == 0:
             return None
@@ -223,7 +213,7 @@ def record_name(self) -> str | None:
         return package_name + naming_element.name
 
     def type_exist_in_tree(self, tree: TypeTree, remaining_tokens: list[str]) -> bool:
-        return self.type_in_tree(tree, remaining_tokens) is not None
+        return tree.type_in_tree(remaining_tokens) is not None
 
     def recursive_imports(self) -> set[str]:
         imports = set(self.proto_file_element.imports)
@@ -355,6 +345,7 @@ def types_tree(self) -> TypeTree:
             token=".",
             children=[],
             source_reference=None,
+            provider=None,
         )
         self.types_tree_recursive(root_tree, 0, "main_schema_file")
         return root_tree
@@ -494,12 +485,13 @@ def to_schema(self) -> str:
 
         return "".join(strings)
 
-    def compare(self, other: ProtobufSchema, result: CompareResult) -> CompareResult:
+    def compare(self, other: ProtobufSchema, result: CompareResult, compare_full_path: bool = False) -> CompareResult:
         return self.proto_file_element.compare(
             other.proto_file_element,
             result,
             self_dependencies=self.dependencies,
             other_dependencies=other.dependencies,
+            compare_full_path=compare_full_path,
         )
 
     def serialize(self) -> str:

karapace/protobuf/service_element.py

@@ -24,8 +24,8 @@ class ServiceElement:
     options: Sequence[OptionElement] | None = None
 
     def with_full_path_expanded(self, type_tree: TypeTree) -> ServiceElement:
-        full_path_options = [option.with_full_path_expanded(type_tree) for option in self.options]
-        full_path_rpcs = [rpc.with_full_path_expanded(type_tree) for rpc in self.rpcs]
+        full_path_options = [option.with_full_path_expanded(type_tree) for option in self.options] if self.options else []
+        full_path_rpcs = [rpc.with_full_path_expanded(type_tree) for rpc in self.rpcs] if self.rpcs else None
         return ServiceElement(
             location=self.location,
             name=self.name,

karapace/protobuf/type_element.py

@@ -6,19 +6,22 @@
 # wire-library/wire-schema/src/commonMain/kotlin/com/squareup/wire/schema/internal/parser/TypeElement.kt
 from __future__ import annotations
 
+from abc import ABC
 from dataclasses import dataclass
 from karapace.protobuf.location import Location
 from karapace.protobuf.type_tree import TypeTree
 from typing import Sequence, TYPE_CHECKING
 
+import abc
+
 if TYPE_CHECKING:
     from karapace.protobuf.compare_result import CompareResult
     from karapace.protobuf.compare_type_storage import CompareTypes
     from karapace.protobuf.option_element import OptionElement
 
 
 @dataclass
-class TypeElement:
+class TypeElement(ABC):
     location: Location
     # https://protobuf.dev/reference/protobuf/proto3-spec/
     # name cannot contain anything else than letters|numbers or `_`
@@ -27,8 +30,9 @@ class TypeElement:
     options: Sequence[OptionElement]
     nested_types: Sequence[TypeElement]
 
+    @abc.abstractmethod
     def with_full_path_expanded(self, type_tree: TypeTree) -> TypeElement:
-        pass
+        ...
 
     def to_schema(self) -> str:
         """Convert the object to valid protobuf syntax.
@@ -45,5 +49,5 @@ def __str__(self) -> str:
         mytype = type(self)
         return f"{mytype}({self.to_schema()})"
 
-    def compare(self, other: TypeElement, result: CompareResult, types: CompareTypes) -> None:
+    def compare(self, other: TypeElement, result: CompareResult, types: CompareTypes, compare_full_path: bool = False) -> None:
         pass

karapace/protobuf/type_tree.py

@@ -4,6 +4,7 @@
 """
 from __future__ import annotations
 
+import copy
 from collections.abc import Iterable, Sequence
 from karapace.dataclasses import default_dataclass
 from typing import Any
@@ -28,7 +29,7 @@ def source_reference_tree_recursive(self) -> Iterable[SourceFileReference | None
         sources = [] if self.source_reference is None else [self.source_reference]
         for child in self.children:
             sources = itertools.chain(sources, child.source_reference_tree())
-        return sources
+        return list(sources)
 
     def source_reference_tree(self) -> Iterable[SourceFileReference]:
         return filter(lambda x: x is not None, self.source_reference_tree_recursive())
@@ -71,6 +72,20 @@ def expand_missing_absolute_path_recursive(self, oldest_import: int) -> Sequence
 
         return []
 
+    @staticmethod
+    def _type_in_tree(tree: TypeTree, remaining_tokens: list[str]) -> TypeTree | None:
+        if remaining_tokens:
+            to_seek = remaining_tokens.pop()
+
+            for child in tree.children:
+                if child.token == to_seek:
+                    return TypeTree._type_in_tree(child, remaining_tokens)
+            return None
+        return tree
+
+    def type_in_tree(self, remaining_tokens: list[str]) -> TypeTree | None:
+        return TypeTree._type_in_tree(self, copy.deepcopy(remaining_tokens))
+
     def expand_missing_absolute_path(self) -> Sequence[str]:
         oldest_import = self.oldest_matching_import()
         expanded_missing_path = self.expand_missing_absolute_path_recursive(oldest_import)
@@ -83,7 +98,7 @@ def expand_missing_absolute_path(self) -> Sequence[str]:
     def is_fully_qualified_type(self) -> bool:
         return len(self.children) == 0
 
-    def represent(self, level=0) -> str:
+    def represent(self, level: int = 0) -> str:
         spacing = " " * 3 * level
         if self.is_fully_qualified_type:
             return f"{spacing}>{self.token}"
@@ -92,3 +107,14 @@ def represent(self, level=0) -> str:
 
     def __repr__(self) -> str:
         return self.represent()
+
+    # useful to see the definitions of the dataclass once defined:
+    # def _pprint(self) -> str:
+    #    return (
+    #        f"TypeTree("
+    #        f"token={self.token},"
+    #        f"children={self.children},"
+    #        f"source_reference={self.source_reference},"
+    #        f"provider={self.provider},"
+    #        f")"
+    #    )

pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
-addopts = -ra --numprocesses auto --import-mode=importlib
 timeout = 90
 timeout_func_only = true

tests/integration/test_dependencies_compatibility_protobuf.py

@@ -6,6 +6,7 @@
 """
 from karapace.client import Client
 from karapace.protobuf.kotlin_wrapper import trim_margin
+from karapace.typing import Subject
 from tests.utils import create_subject_name_factory
 
 import pytest