diff --git a/catalystwan/api/templates/models/cisco_aaa_model.py b/catalystwan/api/templates/models/cisco_aaa_model.py index 0a794f24..18e73182 100644 --- a/catalystwan/api/templates/models/cisco_aaa_model.py +++ b/catalystwan/api/templates/models/cisco_aaa_model.py @@ -18,7 +18,7 @@ class User(FeatureTemplateValidator): class RadiusServer(FeatureTemplateValidator): - model_config = ConfigDict(populate_by_name=True) + model_config = ConfigDict(populate_by_name=True, coerce_numbers_to_str=True) address: str auth_port: int = Field(default=1812, json_schema_extra={"vmanage_key": "auth-port"}) diff --git a/catalystwan/api/templates/models/cisco_secure_internet_gateway.py b/catalystwan/api/templates/models/cisco_secure_internet_gateway.py index 4384148f..71b01362 100644 --- a/catalystwan/api/templates/models/cisco_secure_internet_gateway.py +++ b/catalystwan/api/templates/models/cisco_secure_internet_gateway.py @@ -81,9 +81,9 @@ class Interface(FeatureTemplateValidator): description: Optional[str] = None unnumbered: bool = True address: Optional[ipaddress.IPv4Interface] = None - tunnel_source: ipaddress.IPv4Address = Field(json_schema_extra={"vmanage_key": "tunnel-source"}) - tunnel_source_interface: str = Field(json_schema_extra={"vmanage_key": "tunnel-source-interface"}) - tunnel_route_via: str = Field(json_schema_extra={"vmanage_key": "tunnel-route-via"}) + tunnel_source: Optional[ipaddress.IPv4Address] = Field(default=None, json_schema_extra={"vmanage_key": "tunnel-source"}) + tunnel_source_interface: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "tunnel-source-interface"}) + tunnel_route_via: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "tunnel-route-via"}) tunnel_destination: str = Field(json_schema_extra={"vmanage_key": "tunnel-destination"}) application: Application = Application.SIG tunnel_set: TunnelSet = Field( @@ -157,7 +157,7 @@ class RefreshTimeUnit(str, Enum): class Service(FeatureTemplateValidator): svc_type: SvcType = Field(SvcType.SIG, json_schema_extra={"vmanage_key": "svc-type"}) - interface_pair: List[InterfacePair] = Field(json_schema_extra={"vmanage_key": "interface-pair"}) + interface_pair: List[InterfacePair] = Field(json_schema_extra={"data_path": ["ha-pairs"], "vmanage_key": "interface-pair"}) auth_required: Optional[bool] = Field(False, json_schema_extra={"vmanage_key": "auth-required"}) xff_forward_enabled: Optional[bool] = Field(False, json_schema_extra={"vmanage_key": "xff-forward-enabled"}) ofw_enabled: Optional[bool] = Field(False, json_schema_extra={"vmanage_key": "ofw-enabled"}) @@ -177,12 +177,12 @@ class Service(FeatureTemplateValidator): refresh_time_unit: Optional[RefreshTimeUnit] = Field( RefreshTimeUnit.MINUTE, json_schema_extra={"vmanage_key": "refresh-time-unit"} ) - enabled: Optional[bool] + enabled: Optional[bool] = None block_internet_until_accepted: Optional[bool] = Field( False, json_schema_extra={"vmanage_key": "block-internet-until-accepted"} ) force_ssl_inspection: Optional[bool] = Field(False, json_schema_extra={"vmanage_key": "force-ssl-inspection"}) - timeout: Optional[int] + timeout: Optional[int] = None data_center_primary: Optional[str] = Field("Auto", json_schema_extra={"vmanage_key": "data-center-primary"}) data_center_secondary: Optional[str] = Field("Auto", json_schema_extra={"vmanage_key": "data-center-secondary"}) model_config = ConfigDict(populate_by_name=True) @@ -208,7 +208,7 @@ class CiscoSecureInternetGatewayModel(FeatureTemplate): vpn_id: int = Field(DEFAULT_SIG_VPN_ID, json_schema_extra={"vmanage_key": "vpn-id"}) interface: List[Interface] service: List[Service] - tracker_src_ip: ipaddress.IPv4Interface = Field(json_schema_extra={"vmanage_key": "tracker-src-ip"}) + tracker_src_ip: Optional[ipaddress.IPv4Interface] = Field(default=None, json_schema_extra={"vmanage_key": "tracker-src-ip"}) tracker: Optional[List[Tracker]] = None payload_path: ClassVar[Path] = Path(__file__).parent / "DEPRECATED" diff --git a/catalystwan/api/templates/models/cisco_system.py b/catalystwan/api/templates/models/cisco_system.py index 0d2dd628..6532cc5c 100644 --- a/catalystwan/api/templates/models/cisco_system.py +++ b/catalystwan/api/templates/models/cisco_system.py @@ -54,15 +54,15 @@ class Type(str, Enum): class Tracker(FeatureTemplateValidator): name: str - endpoint_ip: str = Field(json_schema_extra={"vmanage_key": "endpoint-ip"}) - endpoint_ip_transport_port: str = Field( - json_schema_extra={"vmanage_key": "endpoint-ip", "data_path": ["endpoint-ip-transport-port"]} - ) - protocol: Protocol = Field(json_schema_extra={"data_path": ["endpoint-ip-transport-port"]}) - port: int = Field(json_schema_extra={"data_path": ["endpoint-ip-transport-port"]}) - endpoint_dns_name: str = Field(json_schema_extra={"vmanage_key": "endpoint-dns-name"}) - endpoint_api_url: str = Field(json_schema_extra={"vmanage_key": "endpoint-api-url"}) - elements: List[str] + endpoint_ip: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "endpoint-ip"}) + endpoint_ip_transport_port: Optional[str] = Field( + default=None, json_schema_extra={"vmanage_key": "endpoint-ip", "data_path": ["endpoint-ip-transport-port"]} + ) + protocol: Optional[Protocol] = Field(default=None, json_schema_extra={"data_path": ["endpoint-ip-transport-port"]}) + port: Optional[int] = Field(default=None, json_schema_extra={"data_path": ["endpoint-ip-transport-port"]}) + endpoint_dns_name: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "endpoint-dns-name"}) + endpoint_api_url: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "endpoint-api-url"}) + elements: Optional[List[str]] = None boolean: Optional[Boolean] = Boolean.OR threshold: Optional[int] = 300 interval: Optional[int] = 60 diff --git a/catalystwan/api/templates/models/cisco_vpn_interface_model.py b/catalystwan/api/templates/models/cisco_vpn_interface_model.py index 09af6a56..66b82d12 100644 --- a/catalystwan/api/templates/models/cisco_vpn_interface_model.py +++ b/catalystwan/api/templates/models/cisco_vpn_interface_model.py @@ -238,7 +238,7 @@ class Ipv6Vrrp(FeatureTemplateValidator): class CiscoVpnInterfaceModel(FeatureTemplate): model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) - if_name: str = Field(json_schema_extra={"vmanage_key": "if-name"}) + if_name: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "if-name"}) interface_description: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "description"}) poe: Optional[BoolStr] = None ipv4_address: Optional[str] = Field(default=None, json_schema_extra={"data_path": ["ip"], "vmanage_key": "address"}) diff --git a/catalystwan/api/templates/models/cisco_vpn_model.py b/catalystwan/api/templates/models/cisco_vpn_model.py index 04c5d66b..a07741d5 100644 --- a/catalystwan/api/templates/models/cisco_vpn_model.py +++ b/catalystwan/api/templates/models/cisco_vpn_model.py @@ -15,13 +15,13 @@ class Role(str, Enum): class Dns(FeatureTemplateValidator): - dns_addr: str = Field(json_schema_extra={"vmanage_key": "dns-addr"}) + dns_addr: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "dns-addr"}) role: Role = Role.PRIMARY model_config = ConfigDict(populate_by_name=True) class DnsIpv6(FeatureTemplateValidator): - dns_addr: str = Field(json_schema_extra={"vmanage_key": "dns-addr"}) + dns_addr: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "dns-addr"}) role: Optional[Role] = Role.PRIMARY model_config = ConfigDict(populate_by_name=True) @@ -45,8 +45,8 @@ class SvcType(str, Enum): class Service(FeatureTemplateValidator): svc_type: SvcType = Field(json_schema_extra={"vmanage_key": "svc-type"}) - address: List[str] - interface: str + address: Optional[List[str]] = None + interface: Optional[str] = None track_enable: bool = Field(True, json_schema_extra={"vmanage_key": "track-enable"}) model_config = ConfigDict(populate_by_name=True) @@ -67,18 +67,18 @@ class ServiceRoute(FeatureTemplateValidator): class NextHop(FeatureTemplateValidator): - address: str + address: Optional[str] = None distance: Optional[int] = 1 class NextHopWithTrack(FeatureTemplateValidator): - address: str + address: Optional[str] = None distance: Optional[int] = 1 tracker: str class Routev4(FeatureTemplateValidator): - prefix: str + prefix: Optional[str] = None next_hop: Optional[List[NextHop]] = Field( default=None, json_schema_extra={"vmanage_key": "next-hop", "priority_order": ["address", "distance"]} ) @@ -217,9 +217,9 @@ class Overload(str, Enum): class Natpool(FeatureTemplateValidator): name: int - prefix_length: int = Field(json_schema_extra={"vmanage_key": "prefix-length"}) - range_start: str = Field(json_schema_extra={"vmanage_key": "range-start"}) - range_end: str = Field(json_schema_extra={"vmanage_key": "range-end"}) + prefix_length: Optional[int] = Field(default=None, json_schema_extra={"vmanage_key": "prefix-length"}) + range_start: str = Field(default=None, json_schema_extra={"vmanage_key": "range-start"}) + range_end: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "range-end"}) overload: Overload = Overload.TRUE direction: Direction tracker_id: Optional[int] = Field(default=None, json_schema_extra={"vmanage_key": "tracker-id"}) @@ -232,9 +232,9 @@ class StaticNatDirection(str, Enum): class Static(FeatureTemplateValidator): - pool_name: Optional[int] = Field(json_schema_extra={"vmanage_key": "pool-name"}) - source_ip: str = Field(json_schema_extra={"vmanage_key": "source-ip"}) - translate_ip: str = Field(json_schema_extra={"vmanage_key": "translate-ip"}) + pool_name: Optional[int] = Field(default=None, json_schema_extra={"vmanage_key": "pool-name"}) + source_ip: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "source-ip"}) + translate_ip: Optional[str] = Field(default=None, json_schema_extra={"vmanage_key": "translate-ip"}) static_nat_direction: StaticNatDirection = Field(json_schema_extra={"vmanage_key": "static-nat-direction"}) tracker_id: Optional[int] = Field(default=None, json_schema_extra={"vmanage_key": "tracker-id"}) model_config = ConfigDict(populate_by_name=True) diff --git a/catalystwan/integration_tests/test_find_template_values.py b/catalystwan/integration_tests/test_find_template_values.py index c69e7600..ead2692d 100644 --- a/catalystwan/integration_tests/test_find_template_values.py +++ b/catalystwan/integration_tests/test_find_template_values.py @@ -1,6 +1,10 @@ import os import unittest from typing import Any, List, cast +import json + +from pydantic import ValidationError +from catalystwan.exceptions import TemplateTypeError from catalystwan.session import create_manager_session from catalystwan.utils.feature_template.find_template_values import find_template_values @@ -14,11 +18,11 @@ def setUp(self) -> None: username=cast(str, os.environ.get("TEST_VMANAGE_USERNAME")), password=cast(str, os.environ.get("TEST_VMANAGE_PASSWORD")), ) - self.templates = self.session.api.templates._get_feature_templates() - + self.templates = self.session.api.templates._get_feature_templates(summary=False) + def test_find_template_value(self): for template in self.templates: - definition = template.template_definiton + definition = json.loads(template.template_definiton) with self.subTest(template_name=template.name): parsed_values = find_template_values(definition) self.assertFalse( @@ -37,8 +41,9 @@ def is_key_present(self, d: dict, keys: List[Any]): return True if isinstance(value, list): for v in value: - if self.is_key_present(v, keys): - return True + if isinstance(v, dict): + if self.is_key_present(v, keys): + return True return False def tearDown(self) -> None: diff --git a/catalystwan/utils/feature_template/find_template_values.py b/catalystwan/utils/feature_template/find_template_values.py index 16514a82..d281859e 100644 --- a/catalystwan/utils/feature_template/find_template_values.py +++ b/catalystwan/utils/feature_template/find_template_values.py @@ -31,7 +31,6 @@ def find_template_values( path = [] if templated_values is None: templated_values = {} - # if value object is reached, try to extract the value if target_key in template_definition: if template_definition[target_key] == target_key_value_to_ignore: @@ -42,8 +41,23 @@ def find_template_values( field_key = path[-1] # TODO: Handle nested DeviceVariable - if value == "variableName" and (device_specific_variables is not None): - device_specific_variables[field_key] = DeviceVariable(name=template_definition["vipVariableName"]) + if value == "variableName": + if device_specific_variables is not None: + device_specific_variables[field_key] = DeviceVariable(name=template_definition["vipVariableName"]) + elif template_definition["vipType"] == "variable": + if device_specific_variables is not None and template_value: + device_specific_variables[field_key] = DeviceVariable(name=template_value) + elif template_definition["vipObjectType"] == "list": + current_nesting = get_nested_dict(templated_values, path[:-1]) + current_nesting[field_key] = [] + for item in template_value: + if isinstance(item, dict): + if target_key in item: + current_nesting[field_key].append(item[target_key_for_template_value]) + else: + current_nesting[field_key].append(find_template_values(item, device_specific_variables)) + else: + current_nesting[field_key].append(item) elif template_definition["vipObjectType"] != "tree": current_nesting = get_nested_dict(templated_values, path[:-1]) current_nesting[field_key] = template_value @@ -55,10 +69,9 @@ def find_template_values( current_nesting = get_nested_dict(templated_values, path[:-1]) current_nesting[field_key] = [] for item in template_value: - current_nesting[field_key].append( - find_template_values(item, {}, device_specific_variables=device_specific_variables) - ) - + item_value = find_template_values(item, {}, device_specific_variables=device_specific_variables) + if item_value: + current_nesting[field_key].append(item_value) return templated_values # iterate the dict to extract values and assign them to their fields diff --git a/pyproject.toml b/pyproject.toml index 9e4da60f..adb2341e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ typing-extensions = "^4.6.1" parameterized = "^0.8.1" pytest = "^7.1.2" pytest-mock = "^3.7.0" +pytest-subtests = "^0.11.0" isort = "^5.10.1" pre-commit = "^2.19.0" mypy = "^1.0.0"