feat: Implemented AzureSubnet collect

plugins/azure/resoto_plugin_azure/azure_client.py

@@ -149,7 +149,15 @@ def _call(self, spec: AzureApiSpec, **kwargs: Any) -> List[Json]:
         params["api-version"] = _SERIALIZER.query("api_version", spec.version, "str")  # type: ignore
 
         # Construct url
-        path = spec.path.format_map({"subscriptionId": self.subscription_id, "location": self.location, **params})
+        path = spec.path.format_map(
+            {
+                "subscriptionId": self.subscription_id,
+                "location": self.location,
+                "resourceGroupName": kwargs.pop("resourceGroupName", ""),
+                "virtualNetworkName": kwargs.pop("virtualNetworkName", ""),
+                **params,
+            }
+        )
         url = self.client._client.format_url(path)  # pylint: disable=protected-access
 
         # Construct and send request
@@ -165,7 +173,6 @@ def _call(self, spec: AzureApiSpec, **kwargs: Any) -> List[Json]:
             raise HttpResponseError(response=response, error_format=ARMErrorFormat)
 
         # Parse json content
-        # TODO: handle pagination
         js: Union[Json, List[Json]] = response.json()
         if spec.access_path and isinstance(js, dict):
             js = js[spec.access_path]

plugins/azure/resoto_plugin_azure/resource/base.py

@@ -44,16 +44,43 @@ class AzureResource(BaseResource):
     # Which API to call and what to expect in the result.
     api_spec: ClassVar[Optional[AzureApiSpec]] = None
 
-    def resource_subscription_id(self) -> str:
+    def extract_part(self, part: str) -> str:
         """
-        Extracts {subscriptionId} value from a resource ID.
+        Extracts a specific part from a resource ID.
 
-        e.g. /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/...
+        The function takes a resource ID and a specified part to extract, such as 'subscriptionId'
+        or 'resourceGroupName'. The resource ID is expected to follow the Azure Resource Manager
+        path format.
+
+        Example:
+        For the resource ID "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/...",
+        calling extract_part("subscriptionId") would return the value within the curly braces,
+        representing the subscription ID.
+
+        Parameters:
+        - part (str): The part to extract from the resource ID.
 
         Returns:
-        str: The extracted subscription ID.
+        str: The extracted part of the resource ID.
         """
-        return self.id.split("/")[2]
+        id_parts = self.id.split("/")
+
+        if part == "subscriptionId":
+            if "subscriptions" in id_parts:
+                index = id_parts.index("subscriptions")
+                return id_parts[index + 1]
+            else:
+                raise ValueError(f"Id {self.id} does not have any subscriptionId info")
+
+        elif part == "resourceGroupName":
+            if "resourceGroups" in id_parts:
+                index = id_parts.index("resourceGroups")
+                return id_parts[index + 1]
+            else:
+                raise ValueError(f"Id {self.id} does not have any resourceGroupName info")
+
+        else:
+            raise ValueError(f"Value {part} does not have any cases to match")
 
     def delete(self, graph: Graph) -> bool:
         """
@@ -62,7 +89,7 @@ def delete(self, graph: Graph) -> bool:
         Returns:
         bool: True if the resource was successfully deleted; False otherwise.
         """
-        subscription_id = self.resource_subscription_id()
+        subscription_id = self.extract_part("subscriptionId")
         return get_client(subscription_id).delete(self.id)
 
     def delete_tag(self, key: str) -> bool:
@@ -71,7 +98,7 @@ def delete_tag(self, key: str) -> bool:
         This method removes a specific value from a tag associated with a subscription, while keeping the tag itself intact.
         The tag remains on the account, but the specified value will be deleted.
         """
-        subscription_id = self.resource_subscription_id()
+        subscription_id = self.extract_part("subscriptionId")
         return get_client(subscription_id).delete_resource_tag(tag_name=key, resource_id=self.id)
 
     def update_tag(self, key: str, value: str) -> bool:
@@ -80,7 +107,7 @@ def update_tag(self, key: str, value: str) -> bool:
         This method allows for the creation or update of a tag value associated with the specified tag name.
         The tag name must already exist for the operation to be successful.
         """
-        subscription_id = self.resource_subscription_id()
+        subscription_id = self.extract_part("subscriptionId")
         return get_client(subscription_id).update_resource_tag(tag_name=key, tag_value=value, resource_id=self.id)
 
     def pre_process(self, graph_builder: GraphBuilder, source: Json) -> None:

plugins/azure/resoto_plugin_azure/resource/network.py

@@ -2358,9 +2358,23 @@ class AzureDelegation(AzureSubResource):
 
 
 @define(eq=False, slots=False)
-class AzureSubnet(AzureSubResource):
+class AzureSubnet(AzureResource):
     kind: ClassVar[str] = "azure_subnet"
-    mapping: ClassVar[Dict[str, Bender]] = AzureSubResource.mapping | {
+    reference_kinds: ClassVar[ModelReference] = {
+        "successors": {
+            "default": [
+                "azure_nat_gateway",
+                "azure_network_security_group",
+            ]
+        },
+    }
+    mapping: ClassVar[Dict[str, Bender]] = {
+        "id": S("id"),
+        "tags": S("tags", default={}),
+        "name": S("name"),
+        "ctime": K(None),
+        "mtime": K(None),
+        "atime": K(None),
         "address_prefix": S("properties", "addressPrefix"),
         "address_prefixes": S("properties", "addressPrefixes"),
         "application_gateway_ip_configurations": S("properties", "applicationGatewayIPConfigurations")
@@ -2414,6 +2428,12 @@ class AzureSubnet(AzureSubResource):
     service_endpoints: Optional[List[AzureServiceEndpointPropertiesFormat]] = field(default=None, metadata={'description': 'An array of service endpoints.'})  # fmt: skip
     type: Optional[str] = field(default=None, metadata={"description": "Resource type."})
 
+    def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
+        if nat_gateway_id := self.nat_gateway_id:
+            builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureNatGateway, id=nat_gateway_id)
+        if nsg_id := self.network_security_group:
+            builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureNetworkSecurityGroup, id=nsg_id)
+
 
 @define(eq=False, slots=False)
 class AzureFrontendIPConfiguration(AzureSubResource):
@@ -5117,7 +5137,7 @@ class AzureVirtualNetwork(AzureResource, BaseNetwork):
         expect_array=True,
     )
     reference_kinds: ClassVar[ModelReference] = {
-        "successors": {"default": ["azure_subnet", "azure_nat_gateway"]},
+        "successors": {"default": ["azure_subnet"]},
     }
     mapping: ClassVar[Dict[str, Bender]] = {
         "id": S("id"),
@@ -5140,7 +5160,7 @@ class AzureVirtualNetwork(AzureResource, BaseNetwork):
         "ip_allocations": S("properties") >> S("ipAllocations", default=[]) >> ForallBend(S("id")),
         "provisioning_state": S("properties", "provisioningState"),
         "resource_guid": S("properties", "resourceGuid"),
-        "subnets": S("properties", "subnets") >> ForallBend(AzureSubnet.mapping),
+        "_subnet_ids": S("properties", "subnets", default=[]) >> ForallBend(S("id")),
         "virtual_network_peerings": S("properties", "virtualNetworkPeerings")
         >> ForallBend(AzureVirtualNetworkPeering.mapping),
     }
@@ -5158,16 +5178,36 @@ class AzureVirtualNetwork(AzureResource, BaseNetwork):
     ip_allocations: Optional[List[str]] = field(default=None, metadata={'description': 'Array of IpAllocation which reference this VNET.'})  # fmt: skip
     provisioning_state: Optional[str] = field(default=None, metadata={'description': 'The current provisioning state.'})  # fmt: skip
     resource_guid: Optional[str] = field(default=None, metadata={'description': 'The resourceGuid property of the Virtual Network resource.'})  # fmt: skip
-    subnets: Optional[List[AzureSubnet]] = field(default=None, metadata={'description': 'A list of subnets in a Virtual Network.'})  # fmt: skip
+    _subnet_ids: Optional[List[str]] = field(default=None, metadata={'description': 'A list of subnets in a Virtual Network.'})  # fmt: skip
     virtual_network_peerings: Optional[List[AzureVirtualNetworkPeering]] = field(default=None, metadata={'description': 'A list of peerings in a Virtual Network.'})  # fmt: skip
 
+    def post_process(self, graph_builder: GraphBuilder, source: Json) -> None:
+        def collect_subnets() -> None:
+            api_spec = AzureApiSpec(
+                service="network",
+                version="2023-05-01",
+                path="/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets",
+                path_parameters=["subscriptionId", "resourceGroupName", "virtualNetworkName"],
+                query_parameters=["api-version"],
+                access_path="value",
+                expect_array=True,
+            )
+            resource_group_name = self.extract_part("resourceGroupName")
+            virtual_network_name = self.name if self.name else ""
+
+            items = graph_builder.client.list(
+                api_spec,
+                resourceGroupName=resource_group_name,
+                virtualNetworkName=virtual_network_name,
+            )
+            AzureSubnet.collect(items, graph_builder)
+
+        graph_builder.submit_work(collect_subnets)
+
     def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
-        if subnets := self.subnets:
-            for subnet in subnets:
-                if subnet_id := subnet.id:
-                    builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureSubnet, id=subnet_id)
-                if nat_gateway_id := subnet.nat_gateway_id:
-                    builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureNatGateway, id=nat_gateway_id)
+        if subnets := self._subnet_ids:
+            for subnet_id in subnets:
+                builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureSubnet, id=subnet_id)
 
 
 @define(eq=False, slots=False)
@@ -6030,7 +6070,6 @@ class AzureWebApplicationFirewallPolicy(AzureResource):
     AzurePublicIPPrefix,
     AzureRouteFilter,
     AzureSecurityPartnerProvider,
-    # AzureSubnet, # TODO: collect azure subnets properly
     AzureUsage,
     AzureVirtualHub,
     AzureVirtualNetwork,

plugins/azure/test/collector_test.py

@@ -15,5 +15,5 @@ def test_collect(
 ) -> None:
     collector = AzureSubscriptionCollector(config, Cloud(id="azure"), azure_subscription, credentials, core_feedback)
     collector.collect()
-    assert len(collector.graph.nodes) == 467
-    assert len(collector.graph.edges) == 493
+    assert len(collector.graph.nodes) == 471
+    assert len(collector.graph.edges) == 499

plugins/azure/test/files/network/subnets.json

@@ -0,0 +1,20 @@
+{
+    "value": [
+      {
+        "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-1",
+        "name": "subnet1",
+        "properties": {
+          "addressPrefix": "10.0.0.0/16",
+          "provisioningState": "Succeeded"
+        }
+      },
+      {
+        "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/test-2",
+        "name": "subnet2",
+        "properties": {
+          "addressPrefix": "10.0.0.0/16",
+          "provisioningState": "Succeeded"
+        }
+      }
+    ]
+  }

resotocore/resotocore/model/model.py

@@ -1069,7 +1069,7 @@ def check_valid(self, obj: JsonElement, **kwargs: bool) -> ValidationResult:
                         )
             return coerced
         else:
-            raise AttributeError("Kind:{self.fqn} expected a complex type but got this: {obj}")
+            raise AttributeError(f"Kind:{self.fqn} expected a complex type but got this: {obj}")
 
     def coerce_if_required(self, value: JsonElement, **kwargs: bool) -> Optional[Json]:
         if isinstance(value, dict):