From 8614ef15cecafc880c2248063e5754412c3677d6 Mon Sep 17 00:00:00 2001 From: 1101-1 <70093559+1101-1@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:10:39 +0400 Subject: [PATCH] [azure] [feat] Add ResourceGroups and create edges from ResourceGroup to Resource (#1782) --- .../azure/resoto_plugin_azure/collector.py | 11 +++-- .../resoto_plugin_azure/resource/base.py | 30 +++++++++++- plugins/azure/test/collector_test.py | 4 +- .../test/files/resources/resourcegroups.json | 32 +++++++++++++ .../azure/test/files/resources/resources.json | 46 +++++++++++++++++++ plugins/azure/test/resources_test.py | 7 +++ 6 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 plugins/azure/test/files/resources/resourcegroups.json create mode 100644 plugins/azure/test/files/resources/resources.json create mode 100644 plugins/azure/test/resources_test.py diff --git a/plugins/azure/resoto_plugin_azure/collector.py b/plugins/azure/resoto_plugin_azure/collector.py index 122149af3b..02871112a5 100644 --- a/plugins/azure/resoto_plugin_azure/collector.py +++ b/plugins/azure/resoto_plugin_azure/collector.py @@ -6,7 +6,12 @@ from resoto_plugin_azure.config import AzureConfig, AzureCredentials from resoto_plugin_azure.azure_client import AzureClient from resoto_plugin_azure.resource.compute import resources as compute_resources -from resoto_plugin_azure.resource.base import AzureSubscription, GraphBuilder, AzureResource +from resoto_plugin_azure.resource.base import ( + AzureSubscription, + GraphBuilder, + AzureResource, + resources as base_resources, +) from resotolib.baseresources import Cloud, GraphRoot from resotolib.core.actions import CoreFeedback from resotolib.graph import Graph @@ -22,7 +27,7 @@ def resource_with_params(clazz: Type[AzureResource], params: Set[str], includes_ return cp.issubset(params) and (not includes_all or params.issubset(cp)) -all_resources: List[Type[AzureResource]] = compute_resources +all_resources: List[Type[AzureResource]] = compute_resources + base_resources global_resources = [r for r in all_resources if resource_with_params(r, {"subscriptionId"})] regional_resources = [r for r in all_resources if resource_with_params(r, {"subscriptionId", "location"}, True)] @@ -63,7 +68,7 @@ def collect(self) -> None: # wait for all work to finish queue.wait_for_submitted_work() # connect nodes - log.info(f"[Aws:{self.subscription.safe_name}] Connect resources and create edges.") + log.info(f"[Azure:{self.subscription.safe_name}] Connect resources and create edges.") for node, data in list(self.graph.nodes(data=True)): if isinstance(node, AzureResource): node.connect_in_graph(builder, data.get("source", {})) diff --git a/plugins/azure/resoto_plugin_azure/resource/base.py b/plugins/azure/resoto_plugin_azure/resource/base.py index 3afeefbbb3..8eb21a3bc1 100644 --- a/plugins/azure/resoto_plugin_azure/resource/base.py +++ b/plugins/azure/resoto_plugin_azure/resource/base.py @@ -10,7 +10,7 @@ from resoto_plugin_azure.azure_client import AzureApiSpec, AzureClient from resoto_plugin_azure.config import AzureCredentials -from resotolib.baseresources import BaseResource, Cloud, EdgeType, BaseAccount, BaseRegion +from resotolib.baseresources import BaseResource, Cloud, EdgeType, BaseAccount, BaseRegion, ModelReference from resotolib.core.actions import CoreFeedback from resotolib.graph import Graph, EdgeKey from resotolib.json_bender import Bender, bend, S, ForallBend, Bend @@ -188,6 +188,9 @@ class AzureResourceGroup(AzureResource): access_path="value", expect_array=True, ) + reference_kinds: ClassVar[ModelReference] = { + "successors": {"default": ["azure_resource"]}, + } mapping: ClassVar[Dict[str, Bender]] = { "id": S("id"), "tags": S("tags", default={}), @@ -197,6 +200,28 @@ class AzureResourceGroup(AzureResource): } managed_by: Optional[str] = field(default=None, metadata={'description': 'The id of the resource that manages this resource group.'}) # fmt: skip provisioning_state: Optional[str] = field(default=None, metadata={"description": "The resource group properties."}) + _resource_ids_in_group: Optional[List[str]] = None + + def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: + def collect_resources_in_group() -> None: + resources_api_spec = AzureApiSpec( + service="resources", + version="2021-04-01", + path="/subscriptions/{subscriptionId}/resourceGroups/" + f"{self.safe_name}/resources", + path_parameters=["subscriptionId"], + query_parameters=["api-version"], + access_path="value", + expect_array=True, + ) + + self._resource_ids_in_group = [r["id"] for r in graph_builder.client.list(resources_api_spec)] + + graph_builder.submit_work(collect_resources_in_group) + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if resource_ids := self._resource_ids_in_group: + for resource_id in resource_ids: + builder.add_edge(self, edge_type=EdgeType.default, clazz=AzureResource, id=resource_id) @define(eq=False, slots=False) @@ -437,3 +462,6 @@ def with_location(self, location: AzureLocation) -> GraphBuilder: graph_nodes_access=self.graph_nodes_access, graph_edges_access=self.graph_edges_access, ) + + +resources: List[Type[AzureResource]] = [AzureResourceGroup] diff --git a/plugins/azure/test/collector_test.py b/plugins/azure/test/collector_test.py index 9990d4a3ab..e0b120d14b 100644 --- a/plugins/azure/test/collector_test.py +++ b/plugins/azure/test/collector_test.py @@ -15,5 +15,5 @@ def test_collect( ) -> None: collector = AzureSubscriptionCollector(config, Cloud(id="azure"), azure_subscription, credentials, core_feedback) collector.collect() - assert len(collector.graph.nodes) == 60 - assert len(collector.graph.edges) == 78 + assert len(collector.graph.nodes) == 62 + assert len(collector.graph.edges) == 80 diff --git a/plugins/azure/test/files/resources/resourcegroups.json b/plugins/azure/test/files/resources/resourcegroups.json new file mode 100644 index 0000000000..07dd210453 --- /dev/null +++ b/plugins/azure/test/files/resources/resourcegroups.json @@ -0,0 +1,32 @@ +{ + "value": [ + { + "id": "/subscriptions/{subscriptionId}/resourceGroups/myResourceGroup", + "location": "eastus", + "managedBy": "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProvider}/{resourceType}/{resourceName}", + "name": "myResourceGroup", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": { + "tag1": "value1", + "tag2": "value2" + }, + "type": "Microsoft.Resources/resourceGroups" + }, + { + "id": "/subscriptions/{subscriptionId}/resourceGroups/myResourceGroup2", + "location": "westus", + "managedBy": null, + "name": "myResourceGroup2", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": { + "environment": "production", + "department": "IT" + }, + "type": "Microsoft.Resources/resourceGroups" + } + ] +} \ No newline at end of file diff --git a/plugins/azure/test/files/resources/resources.json b/plugins/azure/test/files/resources/resources.json new file mode 100644 index 0000000000..7aaf5d67d1 --- /dev/null +++ b/plugins/azure/test/files/resources/resources.json @@ -0,0 +1,46 @@ +{ + "value": [ + { + "changedTime": null, + "createdTime": null, + "extendedLocation": null, + "id": "/subscriptions/{subscriptionId}/resourceGroups/myResourceGroup/providers/Microsoft.Storage/storageAccounts/myStorageAccount", + "identity": null, + "kind": "Storage", + "location": "eastus", + "managedBy": null, + "name": "myStorageAccount", + "plan": null, + "properties": null, + "provisioningState": null, + "resourceGroup": "myResourceGroup", + "sku": null, + "tags": { + "environment": "production", + "department": "IT" + }, + "type": "Microsoft.Storage/storageAccounts" + }, + { + "changedTime": null, + "createdTime": null, + "extendedLocation": null, + "id": "/subscriptions/{subscriptionId}/resourceGroups/myResourceGroup2/providers/Microsoft.Compute/virtualMachines/myVM", + "identity": null, + "kind": "VirtualMachine", + "location": "westus", + "managedBy": null, + "name": "myVM", + "plan": null, + "properties": null, + "provisioningState": null, + "resourceGroup": "myResourceGroup2", + "sku": null, + "tags": { + "environment": "development", + "department": "IT" + }, + "type": "Microsoft.Compute/virtualMachines" + } + ] +} \ No newline at end of file diff --git a/plugins/azure/test/resources_test.py b/plugins/azure/test/resources_test.py new file mode 100644 index 0000000000..1ad3ff26cc --- /dev/null +++ b/plugins/azure/test/resources_test.py @@ -0,0 +1,7 @@ +from conftest import roundtrip_check +from resoto_plugin_azure.resource.base import GraphBuilder, AzureResourceGroup + + +def test_resource_group(builder: GraphBuilder) -> None: + collected = roundtrip_check(AzureResourceGroup, builder) + assert len(collected) == 2