msggen: fix walk through nested json schemas

Changelog-None

contrib/msggen/msggen/gen/grpc.py

@@ -34,7 +34,7 @@
     "feerate": "Feerate",
     "outputdesc": "OutputDesc",
     "secret": "bytes",
-    "bip340sig": "bytes",
+    "bip340sig": "string",
     "hash": "bytes",
 }
 
@@ -95,8 +95,10 @@ def field2number(self, message_name: TypeName, field):
     def enumerate_fields(self, message_name, fields):
         """Use the meta map to identify which number this field will get.
         """
-        for f in fields:
-            yield (self.field2number(message_name, f), f)
+        enumerated_values = [(self.field2number(message_name, f), f) for f in fields]
+        sorted_enumerated_values = sorted(enumerated_values, key=lambda x: x[0])
+        for i, v in sorted_enumerated_values:
+            yield (i, v)
 
     def enumvar2number(self, typename: TypeName, variant):
         """Find an existing variant number of generate a new one.
@@ -122,8 +124,10 @@ def enumvar2number(self, typename: TypeName, variant):
         return m[typename][variant]
 
     def enumerate_enum(self, typename, variants):
-        for v in variants:
-            yield (self.enumvar2number(typename, v), v)
+        enumerated_values = [(self.enumvar2number(typename, v), v) for v in variants]
+        sorted_enumerated_values = sorted(enumerated_values, key=lambda x: x[0])
+        for i, v in sorted_enumerated_values:
+            yield (i, v)
 
     def gather_types(self, service):
         """Gather all types that might need to be defined.
@@ -249,6 +253,8 @@ def generate_composite(self, prefix, field: CompositeField):
         if field.omit():
             return
 
+        field.sort()
+
         # First pass: generate any sub-fields before we generate the
         # top-level field itself.
         for f in field.fields:
@@ -325,7 +331,6 @@ def generate_composite(self, prefix, field: CompositeField):
                     'hash?': f'c.{name}.map(|v| <Sha256 as AsRef<[u8]>>::as_ref(&v).to_vec())',
                     'secret': f'c.{name}.to_vec()',
                     'secret?': f'c.{name}.map(|v| v.to_vec())',
-
                     'msat_or_any': f'Some(c.{name}.into())',
                     'msat_or_all': f'Some(c.{name}.into())',
                     'msat_or_all?': f'c.{name}.map(|o|o.into())',

contrib/msggen/msggen/gen/grpc2py.py

@@ -97,6 +97,7 @@ def generate_enum(self, prefix, field: EnumField):
             self.converters[field.path] = "str(m.{{name}})"
 
     def generate_composite(self, prefix, field: CompositeField):
+        field.sort()
         if override.get(field.path, "") is None:
             return
         name = field.name.normalized()

contrib/msggen/msggen/gen/rust.py

@@ -35,7 +35,7 @@
     'outputdesc': 'OutputDesc',
     'hash': 'Sha256',
     'secret': 'Secret',
-    'bip340sig': 'Secret',
+    'bip340sig': 'String',
     'integer': 'i64',
 }
 
@@ -110,7 +110,7 @@ def gen_enum(e, meta):
             complete_variants = False
 
     if m != {} and complete_variants:
-        for v in e.variants:
+        for v in sorted(e.variants):
             if v is None:
                 continue
             norm = v.normalized()
@@ -128,12 +128,12 @@ def gen_enum(e, meta):
     """)
 
     if m != {} and complete_variants:
-        for v in e.variants:
+        for v in sorted(e.variants):
             norm = v.normalized()
             # decl += f"    #[serde(rename = \"{v}\")]\n"
             decl += f"    {m[str(v)]} => Ok({e.typename}::{norm}),\n"
     else:
-        for i, v in enumerate(e.variants):
+        for i, v in enumerate(sorted(e.variants)):
             norm = v.normalized()
             # decl += f"    #[serde(rename = \"{v}\")]\n"
             decl += f"    {i} => Ok({e.typename}::{norm}),\n"
@@ -153,7 +153,7 @@ def gen_enum(e, meta):
         fn to_string(&self) -> String {{
             match self {{
     """)
-    for v in e.variants:
+    for v in sorted(e.variants):
         norm = v.normalized()
         decl += f"            {e.typename}::{norm} => \"{norm}\",\n"
     decl += dedent(f"""\
@@ -241,6 +241,7 @@ def gen_composite(c, meta) -> Tuple[str, str]:
     fields = []
     for f in c.fields:
         fields.append(gen_field(f, meta))
+    fields = sorted(fields)
 
     r = "".join([f[1] for f in fields])
 

contrib/msggen/msggen/model.py

@@ -78,6 +78,15 @@ def __init__(
 
         self.type_override: Optional[str] = None
 
+    def __lt__(self, other):
+        return self.path < other.path
+
+    def __eq__(self, other):
+        return self.path == other.path
+
+    def __iter__(self):
+        yield self.path
+
     @property
     def name(self):
         return FieldName(self.path.split(".")[-1])
@@ -199,20 +208,45 @@ def from_js(cls, js, path):
             'else': {'properties': js.get('else', {}).get('properties', [])},
         }
         # Yes, this is ugly, but walking nested dicts always is.
+
+        def merge_dicts(dict1, dict2):
+            merged_dict = {}
+            for key in set(dict1.keys()) | set(dict2.keys()):
+                if key in dict1 and key in dict2:
+                    if isinstance(dict1[key], dict) and isinstance(dict2[key], dict):
+                        merged_dict[key] = merge_dicts(dict1[key], dict2[key])
+                    else:
+                        if isinstance(dict1[key], list) and isinstance(dict2[key], list):
+                            merged_dict[key] = sorted(list(set(dict1[key]).union(set(dict2[key]))))
+                        elif key in dict1:
+                            merged_dict[key] = dict1[key]
+                        else:
+                            merged_dict[key] = dict2[key]
+                elif key in dict1:
+                    merged_dict[key] = dict1[key]
+                else:
+                    merged_dict[key] = dict2[key]
+            return merged_dict
+
         for a in [top] + js.get('allOf', []):
             var = a.get('then', {})
-            props = var.get('properties', None)
+            props = var.get('properties', {})
             if isinstance(props, dict):
                 for k, v in props.items():
-                    if k not in properties:
-                        properties[k] = v
+                    if properties != {}:
+                        if k in properties:
+                            properties[k] = merge_dicts(properties[k], v)
+                        else:
+                            properties[k] = v
             var = a.get('else', {})
-            props = var.get('properties', None)
+            props = var.get('properties', {})
             if isinstance(props, dict):
                 for k, v in props.items():
-                    if k not in properties:
-                        properties[k] = v
-
+                    if properties != {}:
+                        if k in properties:
+                            properties[k] = merge_dicts(properties[k], v)
+                        else:
+                            properties[k] = v
         # Identify required fields
         required = js.get("required", [])
         fields = []
@@ -262,6 +296,9 @@ def from_js(cls, js, path):
             typename, fields, path, js["description"] if "description" in js else "", added=js.get('added', None), deprecated=js.get('deprecated', None)
         )
 
+    def sort(self):
+        self.fields = sorted(self.fields)
+
     def __str__(self):
         fieldnames = ",".join([f.path.split(".")[-1] for f in self.fields])
         return f"CompositeField[name={self.path}, fields=[{fieldnames}]]"
@@ -276,6 +313,12 @@ def __init__(self, variant: Optional[str]):
     def __str__(self):
         return self.variant
 
+    def __lt__(self, other):
+        return self.variant < other.variant
+
+    def __eq__(self, other):
+        return self.variant == other.variant
+
     def normalized(self):
         return self.variant.replace(' ', '_').replace('-', '_').replace("/", "_").upper()