[aws][feat] Make a collection of Ec2 Instance types only for existing instances

plugins/aws/fix_plugin_aws/resource/ec2.py

@@ -6,8 +6,8 @@
 from typing import ClassVar, Dict, Optional, List, Type, Any
 import copy
 
+from boto3.exceptions import Boto3Error
 from attrs import define, field
-from fix_plugin_aws.aws_client import AwsClient
 
 from fix_plugin_aws.resource.base import AwsResource, GraphBuilder, AwsApiSpec, get_client
 from fix_plugin_aws.resource.cloudwatch import (
@@ -22,6 +22,7 @@
 from fix_plugin_aws.resource.s3 import AwsS3Bucket
 from fix_plugin_aws.resource.iam import AwsIamInstanceProfile
 from fix_plugin_aws.utils import ToDict, TagsValue
+from fix_plugin_aws.aws_client import AwsClient
 from fixlib.baseresources import (
     BaseInstance,
     BaseKeyPair,
@@ -392,7 +393,7 @@ class AwsEc2InstanceType(AwsResource, BaseInstanceType):
     _kind_service: ClassVar[Optional[str]] = service_name
     _metadata: ClassVar[Dict[str, Any]] = {"icon": "type", "group": "compute"}
     _aws_metadata: ClassVar[Dict[str, Any]] = {"arn_tpl": "arn:{partition}:ec2:{region}:{account}:instance/{id}"}  # fmt: skip
-    api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes")
+    # api_spec defined in `collect_resource_types` method and collected by AwsEc2Instance
     _reference_kinds: ClassVar[ModelReference] = {
         "successors": {
             "default": ["aws_ec2_instance"],
@@ -457,6 +458,30 @@ class AwsEc2InstanceType(AwsResource, BaseInstanceType):
     auto_recovery_supported: Optional[bool] = field(default=None)
     supported_boot_modes: List[str] = field(factory=list)
 
+    @classmethod
+    def collect_resource_types(cls, builder: GraphBuilder, instance_types: List[str]) -> None:
+        spec = AwsApiSpec(service_name, "describe-instance-types", "InstanceTypes")
+        # Default behavior: in case the class has an ApiSpec, call the api and call collect.
+        log.debug(f"Collecting {cls.__name__} in region {builder.region.name}")
+        try:
+            filters = [{"Name": "instance-type", "Values": instance_types}]
+            items = builder.client.list(
+                aws_service=spec.service,
+                action=spec.api_action,
+                result_name=spec.result_property,
+                expected_errors=spec.expected_errors,
+                Filters=filters,
+            )
+            cls.collect(items, builder)
+        except Boto3Error as e:
+            msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
+            builder.core_feedback.error(msg, log)
+            raise
+        except Exception as e:
+            msg = f"Error while collecting {cls.__name__} in region {builder.region.name}: {e}"
+            builder.core_feedback.info(msg, log)
+            raise
+
     @classmethod
     def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None:
         for js in json:
@@ -468,6 +493,14 @@ def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) ->
                 # we collect instance types in all regions and make the data unique in the builder
                 builder.global_instance_types[it.safe_name] = it
 
+    @classmethod
+    def service_name(cls) -> Optional[str]:
+        return service_name
+
+    @classmethod
+    def called_collect_apis(cls) -> List[AwsApiSpec]:
+        return [AwsApiSpec(service_name, "describe-instance-types")]
+
 
 # endregion
 
@@ -1376,6 +1409,17 @@ class AwsEc2Instance(EC2Taggable, AwsResource, BaseInstance):
     instance_maintenance_options: Optional[str] = field(default=None)
     instance_user_data: Optional[str] = field(default=None)
 
+    @classmethod
+    def collect_resources(cls: Type[AwsResource], builder: GraphBuilder) -> None:
+        super().collect_resources(builder)  # type: ignore # mypy bug: https://github.com/python/mypy/issues/12885
+        ec2_instance_types = set()
+        for instance in builder.nodes(clazz=AwsEc2Instance):
+            ec2_instance_types.add(instance.instance_type)
+        if ec2_instance_types:
+            builder.submit_work(
+                service_name, AwsEc2InstanceType.collect_resource_types, builder, list(ec2_instance_types)
+            )
+
     @classmethod
     def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None:
         def fetch_user_data(instance: AwsEc2Instance) -> None:

plugins/aws/test/graphbuilder_test.py

@@ -17,6 +17,7 @@ def test_instance_type(builder: GraphBuilder) -> None:
         cloud_instance_data, ["aws", instance_type, "pricing", builder.region.id, "linux", "ondemand"]
     )
     eu_builder = builder.for_region(AwsRegion(id="eu-central-1"))
+    builder.global_instance_types[instance_type] = AwsEc2InstanceType(id=instance_type)
     m4l_eu: AwsEc2InstanceType = eu_builder.instance_type(eu_builder.region, instance_type)  # type: ignore
     assert m4l != m4l_eu
     assert m4l_eu == eu_builder.instance_type(eu_builder.region, instance_type)

plugins/aws/test/resources/__init__.py

@@ -178,7 +178,8 @@ def round_trip_for(
     to_collect = [cls] + collect_also if collect_also else [cls]
     builder = build_graph(to_collect, region_name=region_name)
     assert len(builder.graph.nodes) > 0
-    for node, data in builder.graph.nodes(data=True):
+    nodes_to_process = list(builder.graph.nodes(data=True))
+    for node, data in nodes_to_process:
         node.connect_in_graph(builder, data.get("source", {}))
         check_single_node(node)
     first = next(iter(builder.resources_of(cls)))

plugins/aws/test/resources/files/ec2/describe-instance-types__instance_type_m4_large_mac2_metal.json

@@ -0,0 +1,128 @@
+{
+    "InstanceTypes": [
+        {
+            "InstanceType": "m4.large",
+            "CurrentGeneration": true,
+            "FreeTierEligible": false,
+            "SupportedUsageClasses": [
+                "on-demand",
+                "spot"
+            ],
+            "SupportedRootDeviceTypes": [
+                "ebs"
+            ],
+            "SupportedVirtualizationTypes": [
+                "hvm"
+            ],
+            "BareMetal": false,
+            "Hypervisor": "nitro",
+            "ProcessorInfo": {
+                "SupportedArchitectures": [
+                    "x86_64"
+                ],
+                "SustainedClockSpeedInGhz": 3.5
+            },
+            "VCpuInfo": {
+                "DefaultVCpus": 8,
+                "DefaultCores": 4,
+                "DefaultThreadsPerCore": 2,
+                "ValidCores": [
+                    2,
+                    4
+                ],
+                "ValidThreadsPerCore": [
+                    1,
+                    2
+                ]
+            },
+            "MemoryInfo": {
+                "SizeInMiB": 16384
+            },
+            "InstanceStorageSupported": false,
+            "InstanceStorageInfo": {
+                "EbsInfo": {
+                    "EbsStorageSupported": false,
+                    "EbsStorageInfo": {
+                        "VolumeTypes": [
+                            "standard"
+                        ],
+                        "VolumeSizeInGiBMin": 1,
+                        "VolumeSizeInGiBMax": 1024
+                    }
+                },
+                "InstanceStorageSupported": false,
+                "InstanceStorageInfo": {
+                    "VolumeTypes": [
+                        "standard"
+                    ],
+                    "VolumeSizeInGiBMin": 1,
+                    "VolumeSizeInGiBMax": 1024
+                }
+            },
+            "GpuInfo": {
+                "GPUsSupported": false,
+                "GPUSupported": false,
+                "GPUSupportedOnDemand": false,
+                "GPUSupportedSpot": false
+            },
+            "FpgaInfo": {
+                "FPGAsSupported": false,
+                "FPGASupported": false,
+                "FPGASupportedOnDemand": false,
+                "FPGASupportedSpot": false
+            },
+            "InferenceAcceleratorInfo": {
+                "InferenceAcceleratorsSupported": false,
+                "InferenceAcceleratorsSupportedOnDemand": false,
+                "InferenceAcceleratorsSupportedSpot": false
+            },
+            "EbsInfo": {
+                "EbsOptimizedSupport": "default",
+                "EncryptionSupport": "supported",
+                "EbsOptimizedInfo": {
+                    "BaselineBandwidthInMbps": 2500,
+                    "BaselineThroughputInMBps": 312.5,
+                    "BaselineIops": 12000,
+                    "MaximumBandwidthInMbps": 10000,
+                    "MaximumThroughputInMBps": 1250,
+                    "MaximumIops": 40000
+                },
+                "NvmeSupport": "required"
+            },
+            "NetworkInfo": {
+                "NetworkPerformance": "Up to 12.5 Gigabit",
+                "MaximumNetworkInterfaces": 4,
+                "MaximumNetworkCards": 1,
+                "DefaultNetworkCardIndex": 0,
+                "NetworkCards": [
+                    {
+                        "NetworkCardIndex": 0,
+                        "NetworkPerformance": "Up to 12.5 Gigabit",
+                        "MaximumNetworkInterfaces": 4
+                    }
+                ],
+                "Ipv4AddressesPerInterface": 15,
+                "Ipv6AddressesPerInterface": 15,
+                "Ipv6Supported": true,
+                "EnaSupport": "required",
+                "EfaSupported": false,
+                "EncryptionInTransitSupported": true
+            },
+            "PlacementGroupInfo": {
+                "SupportedStrategies": [
+                    "cluster",
+                    "partition",
+                    "spread"
+                ]
+            },
+            "HibernationSupported": false,
+            "BurstablePerformanceSupported": false,
+            "DedicatedHostsSupported": true,
+            "AutoRecoverySupported": true,
+            "SupportedBootModes": [
+                "legacy-bios",
+                "uefi"
+            ]
+        }
+    ]
+}

plugins/aws/test/resources/service_quotas_test.py

@@ -6,7 +6,7 @@
 from fix_plugin_aws.resource.base import AwsResource
 from fix_plugin_aws.aws_client import AwsClient
 from fix_plugin_aws.resource.base import GraphBuilder, AwsRegion
-from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Vpc
+from fix_plugin_aws.resource.ec2 import AwsEc2InstanceType, AwsEc2Instance, AwsEc2Vpc
 from fix_plugin_aws.resource.elbv2 import AwsAlb
 from fix_plugin_aws.resource.iam import AwsIamServerCertificate
 from fix_plugin_aws.resource.service_quotas import AwsServiceQuota, RegionalQuotas
@@ -20,11 +20,10 @@ def test_service_quotas() -> None:
 
 
 def test_instance_type_quotas() -> None:
-    _, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type")
-    AwsEc2InstanceType.collect_resources(builder)
+    _, builder = round_trip_for(AwsServiceQuota, "usage", "quota_type", collect_also=[AwsEc2Instance])
     for _, it in builder.global_instance_types.items():
         builder.add_node(it, {})
-    expect_quotas(builder, 3)
+    expect_quotas(builder, 5)
 
 
 def test_volume_type_quotas() -> None: