Feature template deserialize (#505)

catalystwan/api/templates/feature_template.py

@@ -5,19 +5,50 @@
 import json
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Union, cast
 
 from jinja2 import DebugUndefined, Environment, FileSystemLoader, meta  # type: ignore
 from pydantic import BaseModel, model_validator
 
 from catalystwan.api.templates.device_variable import DeviceVariable
 from catalystwan.utils.device_model import DeviceModel
+from catalystwan.utils.dict import FlattenedDictValue, flatten_dict
+from catalystwan.utils.feature_template.find_template_values import find_template_values
+from catalystwan.utils.pydantic_field import get_extra_field
 
 if TYPE_CHECKING:
     from catalystwan.session import ManagerSession
 
 
-class FeatureTemplate(BaseModel, ABC):
+class FeatureTemplateValidator(BaseModel, ABC):
+    @model_validator(mode="before")
+    @classmethod
+    def map_fields(cls, values: Union[Any, Dict[str, Union[List[FlattenedDictValue], Any]]]):
+        if not isinstance(values, dict):
+            return values
+        for field_name, field_info in cls.model_fields.items():
+            vmanage_key = get_extra_field(field_info, "vmanage_key")
+            if vmanage_key in values:
+                payload_name = vmanage_key
+            elif field_info.alias in values:
+                payload_name = field_info.alias
+            elif field_name in values:
+                payload_name = field_name
+            else:
+                continue
+            data_path = get_extra_field(field_info, "data_path", [])
+            value = values.pop(payload_name)
+            if value and isinstance(value, list) and all([isinstance(v, FlattenedDictValue) for v in value]):
+                for template_value in value:
+                    if template_value.data_path == data_path:
+                        values[field_name] = template_value.value
+                        break
+            else:
+                values[field_name] = value
+        return values
+
+
+class FeatureTemplate(FeatureTemplateValidator, ABC):
     template_name: str
     template_description: str
     device_models: List[DeviceModel] = []
@@ -82,12 +113,11 @@ def get(cls, session: ManagerSession, name: str) -> FeatureTemplate:
         Returns:
             FeatureTemplate: filed out feature template model
         """
-        from catalystwan.utils.feature_template import choose_model, find_template_values
+        from catalystwan.utils.feature_template.choose_model import choose_model
 
         template_info = (
             session.api.templates._get_feature_templates(summary=False).filter(name=name).single_or_default()
         )
-
         template_definition_as_dict = json.loads(cast(str, template_info.template_definiton))
 
         feature_template_model = choose_model(type_value=template_info.template_type)
@@ -96,11 +126,12 @@ def get(cls, session: ManagerSession, name: str) -> FeatureTemplate:
         values_from_template_definition = find_template_values(
             template_definition_as_dict, device_specific_variables=device_specific_variables
         )
+        flattened_values = flatten_dict(values_from_template_definition)
 
         return feature_template_model(
             template_name=template_info.name,
             template_description=template_info.description,
             device_models=[DeviceModel(model) for model in template_info.device_type],
             device_specific_variables=device_specific_variables,
-            **values_from_template_definition,
+            **flattened_values,
         )

catalystwan/api/templates/models/cisco_aaa_model.py

@@ -4,21 +4,21 @@
 from pathlib import Path
 from typing import ClassVar, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
 
-from catalystwan.api.templates.feature_template import FeatureTemplate
+from catalystwan.api.templates.feature_template import FeatureTemplate, FeatureTemplateValidator
 
 
-class User(BaseModel):
+class User(FeatureTemplateValidator):
     name: str
     password: Optional[str] = None
     secret: Optional[str] = None
     privilege: Optional[str] = None
     pubkey_chain: List[str] = Field(default=[], json_schema_extra={"vmanage_key": "pubkey-chain", "vip_type": "ignore"})
 
 
-class RadiusServer(BaseModel):
-    model_config = ConfigDict(populate_by_name=True)
+class RadiusServer(FeatureTemplateValidator):
+    model_config = ConfigDict(populate_by_name=True, coerce_numbers_to_str=True)
 
     address: str
     auth_port: int = Field(default=1812, json_schema_extra={"vmanage_key": "auth-port"})
@@ -31,7 +31,7 @@ class RadiusServer(BaseModel):
     key_type: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "key-type"})
 
 
-class RadiusGroup(BaseModel):
+class RadiusGroup(FeatureTemplateValidator):
     model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True)
 
     group_name: str = Field(json_schema_extra={"vmanage_key": "group-name"})
@@ -46,7 +46,7 @@ class DomainStripping(str, Enum):
     RIGHT_TO_LEFT = "right-to-left"
 
 
-class TacacsServer(BaseModel):
+class TacacsServer(FeatureTemplateValidator):
     model_config = ConfigDict(populate_by_name=True)
 
     address: str
@@ -57,7 +57,7 @@ class TacacsServer(BaseModel):
     key_enum: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "key-enum"})
 
 
-class TacacsGroup(BaseModel):
+class TacacsGroup(FeatureTemplateValidator):
     model_config = ConfigDict(populate_by_name=True)
 
     group_name: str = Field(json_schema_extra={"vmanage_key": "group-name"})

catalystwan/api/templates/models/cisco_bfd_model.py

@@ -4,10 +4,10 @@
 from pathlib import Path
 from typing import ClassVar, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
 
 from catalystwan.api.templates.bool_str import BoolStr
-from catalystwan.api.templates.feature_template import FeatureTemplate
+from catalystwan.api.templates.feature_template import FeatureTemplate, FeatureTemplateValidator
 
 DEFAULT_BFD_COLOR_MULTIPLIER = 7
 DEFAULT_BFD_DSCP = 48
@@ -41,7 +41,7 @@ class ColorType(str, Enum):
     PRIVATE6 = "private6"
 
 
-class Color(BaseModel):
+class Color(FeatureTemplateValidator):
     color: ColorType
     hello_interval: Optional[int] = Field(
         DEFAULT_BFD_HELLO_INTERVAL, json_schema_extra={"vmanage_key": "hello-interval"}

catalystwan/api/templates/models/cisco_bgp_model.py

@@ -4,37 +4,37 @@
 from pathlib import Path
 from typing import ClassVar, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
 
 from catalystwan.api.templates.bool_str import BoolStr
-from catalystwan.api.templates.feature_template import FeatureTemplate
+from catalystwan.api.templates.feature_template import FeatureTemplate, FeatureTemplateValidator
 
 
-class Export(BaseModel):
+class Export(FeatureTemplateValidator):
     asn_ip: str = Field(json_schema_extra={"vmanage_key": "asn-ip"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Import(BaseModel):
+class Import(FeatureTemplateValidator):
     asn_ip: str = Field(json_schema_extra={"vmanage_key": "asn-ip"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class RouteTargetIpv4(BaseModel):
+class RouteTargetIpv4(FeatureTemplateValidator):
     vpn_id: int = Field(json_schema_extra={"vmanage_key": "vpn-id"})
     export: List[Export]
     import_: List[Import] = Field(json_schema_extra={"vmanage_key": "import"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class RouteTargetIpv6(BaseModel):
+class RouteTargetIpv6(FeatureTemplateValidator):
     vpn_id: int = Field(json_schema_extra={"vmanage_key": "vpn-id"})
     export: List[Export]
     import_: List[Import] = Field(json_schema_extra={"vmanage_key": "import"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class MplsInterface(BaseModel):
+class MplsInterface(FeatureTemplateValidator):
     if_name: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "if-name"})
     model_config = ConfigDict(populate_by_name=True)
 
@@ -43,25 +43,25 @@ class AddressFamilyType(str, Enum):
     IPV4_UNICAST = "ipv4-unicast"
 
 
-class AggregateAddress(BaseModel):
+class AggregateAddress(FeatureTemplateValidator):
     prefix: str
     as_set: Optional[BoolStr] = Field(default=None, json_schema_extra={"vmanage_key": "as-set"})
     summary_only: Optional[BoolStr] = Field(default=None, json_schema_extra={"vmanage_key": "summary-only"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Ipv6AggregateAddress(BaseModel):
+class Ipv6AggregateAddress(FeatureTemplateValidator):
     prefix: str
     as_set: Optional[bool] = Field(False, json_schema_extra={"vmanage_key": "as-set"})
     summary_only: Optional[bool] = Field(False, json_schema_extra={"vmanage_key": "summary-only"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Network(BaseModel):
+class Network(FeatureTemplateValidator):
     prefix: str
 
 
-class Ipv6Network(BaseModel):
+class Ipv6Network(FeatureTemplateValidator):
     prefix: str
 
 
@@ -75,13 +75,13 @@ class Protocol(str, Enum):
     NAT = "nat"
 
 
-class Redistribute(BaseModel):
+class Redistribute(FeatureTemplateValidator):
     protocol: Protocol
     route_policy: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "route-policy"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class AddressFamily(BaseModel):
+class AddressFamily(FeatureTemplateValidator):
     family_type: AddressFamilyType = Field(json_schema_extra={"vmanage_key": "family-type"})
     aggregate_address: Optional[List[AggregateAddress]] = Field(
         default=None, json_schema_extra={"vmanage_key": "aggregate-address"}
@@ -112,13 +112,13 @@ class Direction(str, Enum):
     OUT = "out"
 
 
-class RoutePolicy(BaseModel):
+class RoutePolicy(FeatureTemplateValidator):
     direction: Direction
     pol_name: str = Field(json_schema_extra={"vmanage_key": "pol-name"})
     model_config = ConfigDict(populate_by_name=True)
 
 
-class NeighborAddressFamily(BaseModel):
+class NeighborAddressFamily(FeatureTemplateValidator):
     family_type: NeighborFamilyType = Field(json_schema_extra={"vmanage_key": "family-type"})
     prefix_num: Optional[int] = Field(
         default=None, json_schema_extra={"data_path": ["maximum-prefixes"], "vmanage_key": "prefix-num"}
@@ -132,7 +132,7 @@ class NeighborAddressFamily(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Neighbor(BaseModel):
+class Neighbor(FeatureTemplateValidator):
     address: str
     description: Optional[str] = None
     shutdown: Optional[BoolStr] = None
@@ -165,7 +165,7 @@ class IPv6NeighborFamilyType(str, Enum):
     IPV6_UNICAST = "ipv6-unicast"
 
 
-class IPv6NeighborAddressFamily(BaseModel):
+class IPv6NeighborAddressFamily(FeatureTemplateValidator):
     family_type: IPv6NeighborFamilyType = Field(json_schema_extra={"vmanage_key": "family-type"})
     prefix_num: Optional[int] = Field(
         0, json_schema_extra={"data_path": ["maximum-prefixes"], "vmanage_key": "prefix-num"}
@@ -179,7 +179,7 @@ class IPv6NeighborAddressFamily(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Ipv6Neighbor(BaseModel):
+class Ipv6Neighbor(FeatureTemplateValidator):
     address: str
     description: Optional[str] = None
     shutdown: Optional[BoolStr] = None

catalystwan/api/templates/models/cisco_logging_model.py

@@ -4,10 +4,10 @@
 from pathlib import Path
 from typing import ClassVar, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
 
 from catalystwan.api.templates.bool_str import BoolStr
-from catalystwan.api.templates.feature_template import FeatureTemplate
+from catalystwan.api.templates.feature_template import FeatureTemplate, FeatureTemplateValidator
 
 
 class Version(str, Enum):
@@ -20,7 +20,7 @@ class AuthType(str, Enum):
     MUTUAL = "Mutual"
 
 
-class TlsProfile(BaseModel):
+class TlsProfile(FeatureTemplateValidator):
     profile: str
     version: Optional[Version] = Field(Version.TLSV11, json_schema_extra={"data_path": ["tls-version"]})
     auth_type: AuthType = Field(json_schema_extra={"vmanage_key": "auth-type"})
@@ -41,7 +41,7 @@ class Priority(str, Enum):
     EMERGENCY = "emergency"
 
 
-class Server(BaseModel):
+class Server(FeatureTemplateValidator):
     name: str
     vpn: Optional[int] = None
     source_interface: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "source-interface"})
@@ -56,7 +56,7 @@ class Server(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
 
-class Ipv6Server(BaseModel):
+class Ipv6Server(FeatureTemplateValidator):
     name: str
     vpn: Optional[int] = None
     source_interface: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "source-interface"})

catalystwan/api/templates/models/cisco_ntp_model.py

@@ -3,13 +3,13 @@
 from pathlib import Path
 from typing import ClassVar, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
 
 from catalystwan.api.templates.bool_str import BoolStr
-from catalystwan.api.templates.feature_template import FeatureTemplate
+from catalystwan.api.templates.feature_template import FeatureTemplate, FeatureTemplateValidator
 
 
-class Server(BaseModel):
+class Server(FeatureTemplateValidator):
     model_config = ConfigDict(populate_by_name=True)
 
     name: str
@@ -20,7 +20,7 @@ class Server(BaseModel):
     prefer: Optional[BoolStr] = None
 
 
-class Authentication(BaseModel):
+class Authentication(FeatureTemplateValidator):
     model_config = ConfigDict(populate_by_name=True)
 
     number: int

catalystwan/api/templates/models/cisco_omp_model.py

@@ -4,10 +4,10 @@
 from pathlib import Path
 from typing import ClassVar, List, Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import ConfigDict, Field
 
 from catalystwan.api.templates.bool_str import BoolStr
-from catalystwan.api.templates.feature_template import FeatureTemplate
+from catalystwan.api.templates.feature_template import FeatureTemplate, FeatureTemplateValidator
 
 DEFAULT_OMP_HOLDTIME = 60
 DEFAULT_OMP_EOR_TIMER = 300
@@ -32,9 +32,9 @@ class Route(str, Enum):
     EXTERNAL = "external"
 
 
-class IPv4Advertise(BaseModel):
+class IPv4Advertise(FeatureTemplateValidator):
     protocol: IPv4AdvertiseProtocol
-    route: Route
+    route: Optional[Route] = None
 
 
 class IPv6AdvertiseProtocol(str, Enum):
@@ -47,7 +47,7 @@ class IPv6AdvertiseProtocol(str, Enum):
     ISIS = "isis"
 
 
-class IPv6Advertise(BaseModel):
+class IPv6Advertise(FeatureTemplateValidator):
     protocol: IPv6AdvertiseProtocol