From 89abc2cefca5024484edec8b785c63db00458c0a Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Tue, 19 Dec 2023 10:57:43 +0100 Subject: [PATCH 01/10] add ssm --- plugins/aws/resoto_plugin_aws/collector.py | 2 + plugins/aws/resoto_plugin_aws/resource/ssm.py | 86 +++++++++++++++++++ plugins/aws/test/collector_test.py | 4 +- plugins/aws/test/resources/__init__.py | 3 + .../ssm/describe-instance-information.json | 56 ++++++++++++ plugins/aws/test/resources/ssm_test.py | 7 ++ plugins/aws/tools/model_gen.py | 18 +++- plugins/aws/tox.ini | 2 +- resotolib/resotolib/json_bender.py | 24 +++++- 9 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 plugins/aws/resoto_plugin_aws/resource/ssm.py create mode 100644 plugins/aws/test/resources/files/ssm/describe-instance-information.json create mode 100644 plugins/aws/test/resources/ssm_test.py diff --git a/plugins/aws/resoto_plugin_aws/collector.py b/plugins/aws/resoto_plugin_aws/collector.py index 9c1d615687..6b2da5196a 100644 --- a/plugins/aws/resoto_plugin_aws/collector.py +++ b/plugins/aws/resoto_plugin_aws/collector.py @@ -37,6 +37,7 @@ service_quotas, sns, sqs, + ssm, ) from resoto_plugin_aws.resource.base import AwsAccount, AwsApiSpec, AwsRegion, AwsResource, GraphBuilder @@ -87,6 +88,7 @@ + rds.resources + service_quotas.resources + sns.resources + + ssm.resources + sqs.resources + redshift.resources ) diff --git a/plugins/aws/resoto_plugin_aws/resource/ssm.py b/plugins/aws/resoto_plugin_aws/resource/ssm.py new file mode 100644 index 0000000000..92deeae3eb --- /dev/null +++ b/plugins/aws/resoto_plugin_aws/resource/ssm.py @@ -0,0 +1,86 @@ +from datetime import datetime +from typing import ClassVar, Dict, Optional, List, Type + +from attrs import define, field + +from resoto_plugin_aws.resource.base import AwsApiSpec, AwsResource, GraphBuilder +from resoto_plugin_aws.resource.ec2 import AwsEc2Instance +from resoto_plugin_aws.utils import ToDict +from resotolib.baseresources import ModelReference +from resotolib.json_bender import Bender, S, Bend, AsDateString +from resotolib.types import Json + +service_name = "ssm" + + +@define(eq=False, slots=False) +class AwsSSMInstanceAggregatedAssociationOverview: + kind: ClassVar[str] = "aws_ssm_instance_aggregated_association_overview" + mapping: ClassVar[Dict[str, Bender]] = { + "detailed_status": S("DetailedStatus"), + "instance_association_status_aggregated_count": S("InstanceAssociationStatusAggregatedCount"), + } + detailed_status: Optional[str] = field(default=None, metadata={"description": "Detailed status information about the aggregated associations."}) # fmt: skip + instance_association_status_aggregated_count: Optional[Dict[str, int]] = field(default=None, metadata={"description": "The number of associations for the managed node(s)."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsSSMInstanceInformation(AwsResource): + kind: ClassVar[str] = "aws_ssm_instance_information" + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ssm", "describe-instance-information", "InstanceInformationList") + reference_kinds: ClassVar[ModelReference] = { + "successors": {"default": ["aws_ec2_instance"]}, + } + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("id"), + "tags": S("Tags", default=[]) >> ToDict(), + "name": S("Name"), + "instance_id": S("InstanceId"), + "ping_status": S("PingStatus"), + "last_ping": S("LastPingDateTime") >> AsDateString(), + "agent_version": S("AgentVersion"), + "is_latest_version": S("IsLatestVersion"), + "platform_type": S("PlatformType"), + "platform_name": S("PlatformName"), + "platform_version": S("PlatformVersion"), + "activation_id": S("ActivationId"), + "iam_role": S("IamRole"), + "registration_date": S("RegistrationDate") >> AsDateString(), + "resource_type": S("ResourceType"), + "ip_address": S("IPAddress"), + "computer_name": S("ComputerName"), + "association_status": S("AssociationStatus"), + "last_association_execution_date": S("LastAssociationExecutionDate") >> AsDateString(), + "last_successful_association_execution_date": S("LastSuccessfulAssociationExecutionDate") >> AsDateString(), + "association_overview": S("AssociationOverview") >> Bend(AwsSSMInstanceAggregatedAssociationOverview.mapping), + "source_id": S("SourceId"), + "source_type": S("SourceType"), + } + instance_id: Optional[str] = field(default=None, metadata={"description": "The managed node ID."}) # fmt: skip + ping_status: Optional[str] = field(default=None, metadata={"description": "Connection status of SSM Agent. The status Inactive has been deprecated and is no longer in use."}) # fmt: skip + last_ping: Optional[datetime] = field(default=None, metadata={"description": "The date and time when the agent last pinged the Systems Manager service."}) # fmt: skip + agent_version: Optional[str] = field(default=None, metadata={"description": "The version of SSM Agent running on your Linux managed node."}) # fmt: skip + is_latest_version: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the latest version of SSM Agent is running on your Linux managed node. This field doesn't indicate whether or not the latest version is installed on Windows managed nodes, because some older versions of Windows Server use the EC2Config service to process Systems Manager requests."}) # fmt: skip + platform_type: Optional[str] = field(default=None, metadata={"description": "The operating system platform type."}) # fmt: skip + platform_name: Optional[str] = field(default=None, metadata={"description": "The name of the operating system platform running on your managed node."}) # fmt: skip + platform_version: Optional[str] = field(default=None, metadata={"description": "The version of the OS platform running on your managed node."}) # fmt: skip + activation_id: Optional[str] = field(default=None, metadata={"description": "The activation ID created by Amazon Web Services Systems Manager when the server or virtual machine (VM) was registered."}) # fmt: skip + iam_role: Optional[str] = field(default=None, metadata={"description": "The Identity and Access Management (IAM) role assigned to the on-premises Systems Manager managed node. This call doesn't return the IAM role for Amazon Elastic Compute Cloud (Amazon EC2) instances. To retrieve the IAM role for an EC2 instance, use the Amazon EC2 DescribeInstances operation. For information, see DescribeInstances in the Amazon EC2 API Reference or describe-instances in the Amazon Web Services CLI Command Reference."}) # fmt: skip + registration_date: Optional[datetime] = field(default=None, metadata={"description": "The date the server or VM was registered with Amazon Web Services as a managed node."}) # fmt: skip + resource_type: Optional[str] = field(default=None, metadata={"description": "The type of instance. Instances are either EC2 instances or managed instances."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "The name assigned to an on-premises server, edge device, or virtual machine (VM) when it is activated as a Systems Manager managed node. The name is specified as the DefaultInstanceName property using the CreateActivation command. It is applied to the managed node by specifying the Activation Code and Activation ID when you install SSM Agent on the node, as explained in Install SSM Agent for a hybrid environment (Linux) and Install SSM Agent for a hybrid environment (Windows). To retrieve the Name tag of an EC2 instance, use the Amazon EC2 DescribeInstances operation. For information, see DescribeInstances in the Amazon EC2 API Reference or describe-instances in the Amazon Web Services CLI Command Reference."}) # fmt: skip + ip_address: Optional[str] = field(default=None, metadata={"description": "The IP address of the managed node."}) # fmt: skip + computer_name: Optional[str] = field(default=None, metadata={"description": "The fully qualified host name of the managed node."}) # fmt: skip + association_status: Optional[str] = field(default=None, metadata={"description": "The status of the association."}) # fmt: skip + last_association_execution_date: Optional[datetime] = field(default=None, metadata={"description": "The date the association was last run."}) # fmt: skip + last_successful_association_execution_date: Optional[datetime] = field(default=None, metadata={"description": "The last date the association was successfully run."}) # fmt: skip + association_overview: Optional[AwsSSMInstanceAggregatedAssociationOverview] = field(default=None, metadata={"description": "Information about the association."}) # fmt: skip + source_id: Optional[str] = field(default=None, metadata={"description": "The ID of the source resource. For IoT Greengrass devices, SourceId is the Thing name."}) # fmt: skip + source_type: Optional[str] = field(default=None, metadata={"description": "The type of the source resource. For IoT Greengrass devices, SourceType is AWS::IoT::Thing."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if self.resource_type == "EC2Instance" and (instance_id := self.instance_id): + builder.dependant_node(self, clazz=AwsEc2Instance, id=instance_id) + + +resources: List[Type[AwsResource]] = [AwsSSMInstanceInformation] diff --git a/plugins/aws/test/collector_test.py b/plugins/aws/test/collector_test.py index a50e6ec7df..9b4b538194 100644 --- a/plugins/aws/test/collector_test.py +++ b/plugins/aws/test/collector_test.py @@ -33,8 +33,8 @@ def count_kind(clazz: Type[AwsResource]) -> int: # make sure all threads have been joined assert len(threading.enumerate()) == 1 # ensure the correct number of nodes and edges - assert count_kind(AwsResource) == 205 - assert len(account_collector.graph.edges) == 477 + assert count_kind(AwsResource) == 207 + assert len(account_collector.graph.edges) == 483 assert len(account_collector.graph.deferred_edges) == 2 diff --git a/plugins/aws/test/resources/__init__.py b/plugins/aws/test/resources/__init__.py index bb120d03ec..ef9197ab79 100644 --- a/plugins/aws/test/resources/__init__.py +++ b/plugins/aws/test/resources/__init__.py @@ -62,6 +62,9 @@ def arg_string(v: Any) -> str: path = os.path.dirname(__file__) + f"/files/{service}/{action}{vals}.json" return os.path.abspath(path) + def close(self) -> None: + pass + def __getattr__(self, action_name: str) -> Callable[[], Any]: def call_action(*args: Any, **kwargs: Any) -> Any: assert not args, "No arguments allowed!" diff --git a/plugins/aws/test/resources/files/ssm/describe-instance-information.json b/plugins/aws/test/resources/files/ssm/describe-instance-information.json new file mode 100644 index 0000000000..5dc57d7d49 --- /dev/null +++ b/plugins/aws/test/resources/files/ssm/describe-instance-information.json @@ -0,0 +1,56 @@ +{ + "InstanceInformationList": [ + { + "InstanceId": "i-1", + "PingStatus": "ConnectionLost", + "LastPingDateTime": "2023-12-18T16:47:45Z", + "AgentVersion": "foo", + "IsLatestVersion": true, + "PlatformType": "Linux", + "PlatformName": "foo", + "PlatformVersion": "foo", + "ActivationId": "foo", + "IamRole": "foo", + "RegistrationDate": "2023-12-18T16:47:45Z", + "ResourceType": "EC2Instance", + "Name": "foo", + "IPAddress": "foo", + "ComputerName": "foo", + "AssociationStatus": "foo", + "LastAssociationExecutionDate": "2023-12-18T16:47:45Z", + "LastSuccessfulAssociationExecutionDate": "2023-12-18T16:47:45Z", + "AssociationOverview": { + "DetailedStatus": "foo", + "InstanceAssociationStatusAggregatedCount": { + "0": 123, + "1": 123, + "2": 123 + } + }, + "SourceId": "foo", + "SourceType": "AWS::IoT::Thing" + }, + { + "AgentVersion": "2.3.871.0", + "AssociationOverview": { + "DetailedStatus": "Failed", + "InstanceAssociationStatusAggregatedCount": { + "Failed": 1, + "Success": 1 + } + }, + "AssociationStatus": "Failed", + "ComputerName": "WIN-11RMS222RPK.WORKGROUP", + "IPAddress": "203.0.113.0", + "InstanceId": "i-123", + "IsLatestVersion": false, + "LastAssociationExecutionDate": 1582242019, + "LastPingDateTime": 1582242018.094, + "PingStatus": "Online", + "PlatformName": "Microsoft Windows Server 2008 R2 Datacenter", + "PlatformType": "Windows", + "PlatformVersion": "6.1.7601", + "ResourceType": "EC2Instance" + } + ] +} diff --git a/plugins/aws/test/resources/ssm_test.py b/plugins/aws/test/resources/ssm_test.py new file mode 100644 index 0000000000..6e4c15e269 --- /dev/null +++ b/plugins/aws/test/resources/ssm_test.py @@ -0,0 +1,7 @@ +from resoto_plugin_aws.resource.ssm import AwsSSMInstanceInformation +from test.resources import round_trip_for + + +def test_queues() -> None: + first, builder = round_trip_for(AwsSSMInstanceInformation) + assert len(builder.resources_of(AwsSSMInstanceInformation)) == 2 diff --git a/plugins/aws/tools/model_gen.py b/plugins/aws/tools/model_gen.py index f733c11fcf..9f0ea8a5fa 100644 --- a/plugins/aws/tools/model_gen.py +++ b/plugins/aws/tools/model_gen.py @@ -5,6 +5,7 @@ import boto3 from attrs import define +from bs4 import BeautifulSoup from botocore.model import ServiceModel, StringShape, ListShape, Shape, StructureShape, MapShape from jsons import pascalcase @@ -25,7 +26,8 @@ class AwsProperty: def assignment(self) -> str: default = self.field_default or ("factory=list" if self.is_array else "default=None") - return f"field({default})" + description = BeautifulSoup(self.description, "lxml").get_text().strip() + return f'field({default}, metadata={{"description": "{description}"}}) # fmt: skip' def type_string(self) -> str: if self.is_array: @@ -988,15 +990,23 @@ def default_imports() -> str: # prop_prefix="configuration_recorder_", # ), ], + "ssm": [ + AwsResotoModel( + "describe-instance-information", + "InstanceInformationList", + "InstanceInformation", + prefix="SSM", + ), + ], } if __name__ == "__main__": """print some test data""" - # print(json.dumps(create_test_response("logs", "describe-log-groups"), indent=2)) + print(json.dumps(create_test_response("ssm", "describe-instance-information"), indent=2)) """print the class models""" # print(default_imports()) for model in all_models(): - # pass - print(model.to_class()) + pass + # print(model.to_class()) diff --git a/plugins/aws/tox.ini b/plugins/aws/tox.ini index 5dcc108a52..ab119d8a51 100644 --- a/plugins/aws/tox.ini +++ b/plugins/aws/tox.ini @@ -4,7 +4,7 @@ env_list = syntax, tests, black, mypy [flake8] max-line-length=120 exclude = .git,.tox,__pycache__,.idea,.pytest_cache -ignore=F403, F405, E722, N806, N813, E266, W503, E203, F811 +ignore=F403, F405, E722, N806, N813, E266, W503, E203, F811, E501 [pytest] testpaths = test diff --git a/resotolib/resotolib/json_bender.py b/resotolib/resotolib/json_bender.py index 182db71f00..aec7d4f03f 100644 --- a/resotolib/resotolib/json_bender.py +++ b/resotolib/resotolib/json_bender.py @@ -390,7 +390,29 @@ def __init__(self, date_format: str = "%Y-%m-%dT%H:%M:%SZ", **kwargs: Any): self._format = date_format def execute(self, source: Any) -> Any: - return datetime.strptime(source, self._format) if isinstance(source, str) else source + if isinstance(source, str): + return datetime.strptime(source, self._format) + elif isinstance(source, (int, float)): + return datetime.utcfromtimestamp(source) + else: + return source + + +class AsDateString(Bender): + """ + Parse a given input timestamp as date string. + The format of the date needs to be defined. + """ + + def __init__(self, out_format: str = "%Y-%m-%dT%H:%M:%SZ", **kwargs: Any): + super().__init__(**kwargs) + self._out_format = out_format + + def execute(self, source: Any) -> Any: + if isinstance(source, (int, float)): + return datetime.utcfromtimestamp(source).strftime(self._out_format) + else: + return source class ListOp(Bender, ABC): From e9bf1815c3c9214c673f96a366cf15aba32267cb Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Tue, 19 Dec 2023 11:31:04 +0100 Subject: [PATCH 02/10] add ecr --- plugins/aws/resoto_plugin_aws/collector.py | 2 + plugins/aws/resoto_plugin_aws/resource/ecr.py | 74 +++++++++++++++++++ plugins/aws/test/collector_test.py | 4 +- plugins/aws/test/resources/ecr_test.py | 7 ++ .../files/ecr/describe-repositories.json | 50 +++++++++++++ .../tools/{model_gen.py => aws_model_gen.py} | 15 ++-- 6 files changed, 142 insertions(+), 10 deletions(-) create mode 100644 plugins/aws/resoto_plugin_aws/resource/ecr.py create mode 100644 plugins/aws/test/resources/ecr_test.py create mode 100644 plugins/aws/test/resources/files/ecr/describe-repositories.json rename plugins/aws/tools/{model_gen.py => aws_model_gen.py} (98%) diff --git a/plugins/aws/resoto_plugin_aws/collector.py b/plugins/aws/resoto_plugin_aws/collector.py index 6b2da5196a..1f7d71303a 100644 --- a/plugins/aws/resoto_plugin_aws/collector.py +++ b/plugins/aws/resoto_plugin_aws/collector.py @@ -38,6 +38,7 @@ sns, sqs, ssm, + ecr, ) from resoto_plugin_aws.resource.base import AwsAccount, AwsApiSpec, AwsRegion, AwsResource, GraphBuilder @@ -76,6 +77,7 @@ + ec2.resources + efs.resources + ecs.resources + + ecr.resources + eks.resources + elasticbeanstalk.resources + elasticache.resources diff --git a/plugins/aws/resoto_plugin_aws/resource/ecr.py b/plugins/aws/resoto_plugin_aws/resource/ecr.py new file mode 100644 index 0000000000..63f53d3079 --- /dev/null +++ b/plugins/aws/resoto_plugin_aws/resource/ecr.py @@ -0,0 +1,74 @@ +from datetime import datetime +from typing import ClassVar, Dict, Optional, List, Type + +from attrs import define, field + +from resoto_plugin_aws.resource.base import AwsResource, AwsApiSpec +from resoto_plugin_aws.utils import TagsValue, ToDict +from resotolib.json_bender import Bender, S, Bend + +service_name = "ecs" + + +@define(eq=False, slots=False) +class AwsEcrEncryptionConfiguration: + kind: ClassVar[str] = "aws_ecr_encryption_configuration" + mapping: ClassVar[Dict[str, Bender]] = {"encryption_type": S("encryptionType"), "kms_key": S("kmsKey")} + encryption_type: Optional[str] = field(default=None, metadata={"description": "The encryption type to use. If you use the KMS encryption type, the contents of the repository will be encrypted using server-side encryption with Key Management Service key stored in KMS. When you use KMS to encrypt your data, you can either use the default Amazon Web Services managed KMS key for Amazon ECR, or specify your own KMS key, which you already created. For more information, see Protecting data using server-side encryption with an KMS key stored in Key Management Service (SSE-KMS) in the Amazon Simple Storage Service Console Developer Guide. If you use the AES256 encryption type, Amazon ECR uses server-side encryption with Amazon S3-managed encryption keys which encrypts the images in the repository using an AES-256 encryption algorithm. For more information, see Protecting data using server-side encryption with Amazon S3-managed encryption keys (SSE-S3) in the Amazon Simple Storage Service Console Developer Guide."}) # fmt: skip + kms_key: Optional[str] = field(default=None, metadata={"description": "If you use the KMS encryption type, specify the KMS key to use for encryption. The alias, key ID, or full ARN of the KMS key can be specified. The key must exist in the same Region as the repository. If no key is specified, the default Amazon Web Services managed KMS key for Amazon ECR will be used."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsEcrRepository(AwsResource): + kind: ClassVar[str] = "aws_ecr_repository" + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ecr", "describe-repositories", "repositories") + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("repositoryName"), + "tags": S("Tags", default=[]) >> ToDict(), + "name": S("repositoryName"), + "ctime": S("createdAt"), + "repository_arn": S("repositoryArn"), + "registry_id": S("registryId"), + "repository_uri": S("repositoryUri"), + "image_tag_mutability": S("imageTagMutability"), + "image_scanning_configuration": S("imageScanningConfiguration", "scanOnPush"), + "encryption_configuration": S("encryptionConfiguration") >> Bend(AwsEcrEncryptionConfiguration.mapping), + } + repository_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) that identifies the repository. The ARN contains the arn:aws:ecr namespace, followed by the region of the repository, Amazon Web Services account ID of the repository owner, repository namespace, and repository name. For example, arn:aws:ecr:region:012345678910:repository-namespace/repository-name."}) # fmt: skip + registry_id: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services account ID associated with the registry that contains the repository."}) # fmt: skip + repository_uri: Optional[str] = field(default=None, metadata={"description": "The URI for the repository. You can use this URI for container image push and pull operations."}) # fmt: skip + image_tag_mutability: Optional[str] = field(default=None, metadata={"description": "The tag mutability setting for the repository."}) # fmt: skip + image_scanning_configuration: Optional[bool] = field(default=None, metadata={"description": "The image scanning configuration for a repository."}) # fmt: skip + encryption_configuration: Optional[AwsEcrEncryptionConfiguration] = field(default=None, metadata={"description": "The encryption configuration for the repository. This determines how the contents of your repository are encrypted at rest."}) # fmt: skip + + +# @define(eq=False, slots=False) +# class AwsEcrImageIdentifier: +# kind: ClassVar[str] = "aws_ecr_image_identifier" +# mapping: ClassVar[Dict[str, Bender]] = {"image_digest": S("imageDigest"), "image_tag": S("imageTag")} +# image_digest: Optional[str] = field(default=None, metadata={"description": "The sha256 digest of the image manifest."}) # fmt: skip +# image_tag: Optional[str] = field(default=None, metadata={"description": "The tag used for the image."}) # fmt: skip +# +# +# @define(eq=False, slots=False) +# class AwsEcrImage(AwsResource): +# kind: ClassVar[str] = "aws_ecr_image" +# api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("ecr", "describe-images", "images") +# mapping: ClassVar[Dict[str, Bender]] = { +# "id": S("id"), +# "tags": S("Tags", default=[]) >> ToDict(), +# "name": S("Tags", default=[]) >> TagsValue("Name"), +# "registry_id": S("registryId"), +# "repository_name": S("repositoryName"), +# "image_id": S("imageId") >> Bend(AwsEcrImageIdentifier.mapping), +# "image_manifest": S("imageManifest"), +# "image_manifest_media_type": S("imageManifestMediaType"), +# } +# registry_id: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services account ID associated with the registry containing the image."}) # fmt: skip +# repository_name: Optional[str] = field(default=None, metadata={"description": "The name of the repository associated with the image."}) # fmt: skip +# image_id: Optional[AwsEcrImageIdentifier] = field(default=None, metadata={"description": "An object containing the image tag and image digest associated with an image."}) # fmt: skip +# image_manifest: Optional[str] = field(default=None, metadata={"description": "The image manifest associated with the image."}) # fmt: skip +# image_manifest_media_type: Optional[str] = field(default=None, metadata={"description": "The manifest media type of the image."}) # fmt: skip + + +resources: List[Type[AwsResource]] = [AwsEcrRepository] diff --git a/plugins/aws/test/collector_test.py b/plugins/aws/test/collector_test.py index 9b4b538194..2c88c17f31 100644 --- a/plugins/aws/test/collector_test.py +++ b/plugins/aws/test/collector_test.py @@ -33,8 +33,8 @@ def count_kind(clazz: Type[AwsResource]) -> int: # make sure all threads have been joined assert len(threading.enumerate()) == 1 # ensure the correct number of nodes and edges - assert count_kind(AwsResource) == 207 - assert len(account_collector.graph.edges) == 483 + assert count_kind(AwsResource) == 210 + assert len(account_collector.graph.edges) == 486 assert len(account_collector.graph.deferred_edges) == 2 diff --git a/plugins/aws/test/resources/ecr_test.py b/plugins/aws/test/resources/ecr_test.py new file mode 100644 index 0000000000..d173e2e4e0 --- /dev/null +++ b/plugins/aws/test/resources/ecr_test.py @@ -0,0 +1,7 @@ +from resoto_plugin_aws.resource.ecr import AwsEcrRepository +from test.resources import round_trip_for + + +def test_ecr_repositories() -> None: + first, builder = round_trip_for(AwsEcrRepository) + assert len(builder.resources_of(AwsEcrRepository)) == 3 diff --git a/plugins/aws/test/resources/files/ecr/describe-repositories.json b/plugins/aws/test/resources/files/ecr/describe-repositories.json new file mode 100644 index 0000000000..f2d30cfc6c --- /dev/null +++ b/plugins/aws/test/resources/files/ecr/describe-repositories.json @@ -0,0 +1,50 @@ +{ + "repositories": [ + { + "repositoryArn": "arn:aws:ecr:us-east-1:test:repository/ubuntu", + "registryId": "foo", + "repositoryName": "ubuntu", + "repositoryUri": "foo", + "createdAt": "2023-12-19T10:21:23Z", + "imageTagMutability": "IMMUTABLE", + "imageScanningConfiguration": { + "scanOnPush": true + }, + "encryptionConfiguration": { + "encryptionType": "KMS", + "kmsKey": "foo" + } + }, + { + "repositoryArn": "foo", + "registryId": "foo", + "repositoryName": "foo", + "repositoryUri": "foo", + "createdAt": "2023-12-19T10:21:23Z", + "imageTagMutability": "IMMUTABLE", + "imageScanningConfiguration": { + "scanOnPush": true + }, + "encryptionConfiguration": { + "encryptionType": "KMS", + "kmsKey": "foo" + } + }, + { + "repositoryArn": "foo", + "registryId": "foo", + "repositoryName": "foo", + "repositoryUri": "foo", + "createdAt": "2023-12-19T10:21:23Z", + "imageTagMutability": "IMMUTABLE", + "imageScanningConfiguration": { + "scanOnPush": true + }, + "encryptionConfiguration": { + "encryptionType": "KMS", + "kmsKey": "foo" + } + } + ], + "nextToken": "foo" +} diff --git a/plugins/aws/tools/model_gen.py b/plugins/aws/tools/aws_model_gen.py similarity index 98% rename from plugins/aws/tools/model_gen.py rename to plugins/aws/tools/aws_model_gen.py index 9f0ea8a5fa..46c9bad00e 100644 --- a/plugins/aws/tools/model_gen.py +++ b/plugins/aws/tools/aws_model_gen.py @@ -5,7 +5,7 @@ import boto3 from attrs import define -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # pip install beautifulsoup4 lxml from botocore.model import ServiceModel, StringShape, ListShape, Shape, StructureShape, MapShape from jsons import pascalcase @@ -713,6 +713,10 @@ def default_imports() -> str: # prefix="Alb", # ), ], + "ecr": [ + # AwsResotoModel("describe-repositories", "repositories", "Repository", prefix="Ecr"), + AwsResotoModel("describe-images", "images", "Image", prefix="Ecr"), + ], "eks": [ # AwsResotoModel("list-clusters", "clusters", "Cluster", prefix="Eks", prop_prefix="cluster_"), # AwsResotoModel("list-nodegroups", "nodegroup", "Nodegroup", prefix="Eks", prop_prefix="group_"), @@ -991,19 +995,14 @@ def default_imports() -> str: # ), ], "ssm": [ - AwsResotoModel( - "describe-instance-information", - "InstanceInformationList", - "InstanceInformation", - prefix="SSM", - ), + # AwsResotoModel("describe-instance-information", "InstanceInformationList", "InstanceInformation", prefix="SSM"), ], } if __name__ == "__main__": """print some test data""" - print(json.dumps(create_test_response("ssm", "describe-instance-information"), indent=2)) + print(json.dumps(create_test_response("ecr", "describe-repositories"), indent=2)) """print the class models""" # print(default_imports()) From 9147761f8f75ad4cf8ee55d7492f3dc11dacad9b Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Tue, 19 Dec 2023 13:57:08 +0100 Subject: [PATCH 03/10] add secretsmanager --- plugins/aws/resoto_plugin_aws/collector.py | 2 + plugins/aws/resoto_plugin_aws/resource/ecr.py | 4 +- .../resource/secretsmanager.py | 73 ++++++++++++++++++ plugins/aws/test/collector_test.py | 4 +- .../files/secretsmanager/list-secrets.json | 74 +++++++++++++++++++ .../aws/test/resources/secretsmanager_test.py | 7 ++ plugins/aws/tools/aws_model_gen.py | 8 +- .../resotocore/report/inspector_service.py | 2 +- .../static/report/checks/aws/aws_ecr.json | 21 ++++++ .../report/checks/aws/aws_sagemaker.json | 24 ++++++ 10 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 plugins/aws/resoto_plugin_aws/resource/secretsmanager.py create mode 100644 plugins/aws/test/resources/files/secretsmanager/list-secrets.json create mode 100644 plugins/aws/test/resources/secretsmanager_test.py create mode 100644 resotocore/resotocore/static/report/checks/aws/aws_ecr.json create mode 100644 resotocore/resotocore/static/report/checks/aws/aws_sagemaker.json diff --git a/plugins/aws/resoto_plugin_aws/collector.py b/plugins/aws/resoto_plugin_aws/collector.py index 1f7d71303a..b9b7833b7f 100644 --- a/plugins/aws/resoto_plugin_aws/collector.py +++ b/plugins/aws/resoto_plugin_aws/collector.py @@ -39,6 +39,7 @@ sqs, ssm, ecr, + secretsmanager, ) from resoto_plugin_aws.resource.base import AwsAccount, AwsApiSpec, AwsRegion, AwsResource, GraphBuilder @@ -88,6 +89,7 @@ + kms.resources + lambda_.resources + rds.resources + + secretsmanager.resources + service_quotas.resources + sns.resources + ssm.resources diff --git a/plugins/aws/resoto_plugin_aws/resource/ecr.py b/plugins/aws/resoto_plugin_aws/resource/ecr.py index 63f53d3079..ee148bdea6 100644 --- a/plugins/aws/resoto_plugin_aws/resource/ecr.py +++ b/plugins/aws/resoto_plugin_aws/resource/ecr.py @@ -31,14 +31,14 @@ class AwsEcrRepository(AwsResource): "registry_id": S("registryId"), "repository_uri": S("repositoryUri"), "image_tag_mutability": S("imageTagMutability"), - "image_scanning_configuration": S("imageScanningConfiguration", "scanOnPush"), + "image_scan_on_push": S("imageScanningConfiguration", "scanOnPush"), "encryption_configuration": S("encryptionConfiguration") >> Bend(AwsEcrEncryptionConfiguration.mapping), } repository_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) that identifies the repository. The ARN contains the arn:aws:ecr namespace, followed by the region of the repository, Amazon Web Services account ID of the repository owner, repository namespace, and repository name. For example, arn:aws:ecr:region:012345678910:repository-namespace/repository-name."}) # fmt: skip registry_id: Optional[str] = field(default=None, metadata={"description": "The Amazon Web Services account ID associated with the registry that contains the repository."}) # fmt: skip repository_uri: Optional[str] = field(default=None, metadata={"description": "The URI for the repository. You can use this URI for container image push and pull operations."}) # fmt: skip image_tag_mutability: Optional[str] = field(default=None, metadata={"description": "The tag mutability setting for the repository."}) # fmt: skip - image_scanning_configuration: Optional[bool] = field(default=None, metadata={"description": "The image scanning configuration for a repository."}) # fmt: skip + image_scan_on_push: Optional[bool] = field(default=None, metadata={"description": "The image is scanned on every push."}) # fmt: skip encryption_configuration: Optional[AwsEcrEncryptionConfiguration] = field(default=None, metadata={"description": "The encryption configuration for the repository. This determines how the contents of your repository are encrypted at rest."}) # fmt: skip diff --git a/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py b/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py new file mode 100644 index 0000000000..8825549118 --- /dev/null +++ b/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py @@ -0,0 +1,73 @@ +from datetime import datetime +from typing import ClassVar, Dict, Optional, List, Type + +from attrs import define, field + +from resoto_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder +from resoto_plugin_aws.resource.kms import AwsKmsKey +from resoto_plugin_aws.utils import ToDict +from resotolib.json_bender import Bender, S, Bend +from resotolib.types import Json + + +@define(eq=False, slots=False) +class AwsSecretsManagerRotationRulesType: + kind: ClassVar[str] = "aws_secrets_manager_rotation_rules_type" + mapping: ClassVar[Dict[str, Bender]] = { + "automatically_after_days": S("AutomaticallyAfterDays"), + "duration": S("Duration"), + "schedule_expression": S("ScheduleExpression"), + } + automatically_after_days: Optional[int] = field(default=None, metadata={"description": "The number of days between rotations of the secret. You can use this value to check that your secret meets your compliance guidelines for how often secrets must be rotated. If you use this field to set the rotation schedule, Secrets Manager calculates the next rotation date based on the previous rotation. Manually updating the secret value by calling PutSecretValue or UpdateSecret is considered a valid rotation. In DescribeSecret and ListSecrets, this value is calculated from the rotation schedule after every successful rotation. In RotateSecret, you can set the rotation schedule in RotationRules with AutomaticallyAfterDays or ScheduleExpression, but not both. To set a rotation schedule in hours, use ScheduleExpression."}) # fmt: skip + duration: Optional[str] = field(default=None, metadata={"description": "The length of the rotation window in hours, for example 3h for a three hour window. Secrets Manager rotates your secret at any time during this window. The window must not extend into the next rotation window or the next UTC day. The window starts according to the ScheduleExpression. If you don't specify a Duration, for a ScheduleExpression in hours, the window automatically closes after one hour. For a ScheduleExpression in days, the window automatically closes at the end of the UTC day. For more information, including examples, see Schedule expressions in Secrets Manager rotation in the Secrets Manager Users Guide."}) # fmt: skip + schedule_expression: Optional[str] = field(default=None, metadata={"description": "A cron() or rate() expression that defines the schedule for rotating your secret. Secrets Manager rotation schedules use UTC time zone. Secrets Manager rotates your secret any time during a rotation window. Secrets Manager rate() expressions represent the interval in hours or days that you want to rotate your secret, for example rate(12 hours) or rate(10 days). You can rotate a secret as often as every four hours. If you use a rate() expression, the rotation window starts at midnight. For a rate in hours, the default rotation window closes after one hour. For a rate in days, the default rotation window closes at the end of the day. You can set the Duration to change the rotation window. The rotation window must not extend into the next UTC day or into the next rotation window. You can use a cron() expression to create a rotation schedule that is more detailed than a rotation interval. For more information, including examples, see Schedule expressions in Secrets Manager rotation in the Secrets Manager Users Guide. For a cron expression that represents a schedule in hours, the default rotation window closes after one hour. For a cron expression that represents a schedule in days, the default rotation window closes at the end of the day. You can set the Duration to change the rotation window. The rotation window must not extend into the next UTC day or into the next rotation window."}) # fmt: skip + + +@define(eq=False, slots=False) +class AwsSecretsManagerSecret(AwsResource): + kind: ClassVar[str] = "aws_secrets_manager_secret" + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("secretsmanager", "list-secrets", "SecretList") + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("Name"), + "tags": S("Tags", default=[]) >> ToDict(), + "name": S("Name"), + "ctime": S("CreatedDate"), + "mtime": S("LastChangedDate"), + "atime": S("LastAccessedDate"), + "arn": S("ARN"), + "description": S("Description"), + "rotation_enabled": S("RotationEnabled"), + "rotation_lambda_arn": S("RotationLambdaARN"), + "rotation_rules": S("RotationRules") >> Bend(AwsSecretsManagerRotationRulesType.mapping), + "last_rotated_date": S("LastRotatedDate"), + "last_changed_date": S("LastChangedDate"), + "last_accessed_date": S("LastAccessedDate"), + "deleted_date": S("DeletedDate"), + "next_rotation_date": S("NextRotationDate"), + "secret_versions_to_stages": S("SecretVersionsToStages"), + "owning_service": S("OwningService"), + "created_date": S("CreatedDate"), + "primary_region": S("PrimaryRegion"), + } + arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the secret."}) # fmt: skip + name: Optional[str] = field(default=None, metadata={"description": "The friendly name of the secret."}) # fmt: skip + description: Optional[str] = field(default=None, metadata={"description": "The user-provided description of the secret."}) # fmt: skip + rotation_enabled: Optional[bool] = field(default=None, metadata={"description": "Indicates whether automatic, scheduled rotation is enabled for this secret."}) # fmt: skip + rotation_lambda_arn: Optional[str] = field(default=None, metadata={"description": "The ARN of an Amazon Web Services Lambda function invoked by Secrets Manager to rotate and expire the secret either automatically per the schedule or manually by a call to RotateSecret ."}) # fmt: skip + rotation_rules: Optional[AwsSecretsManagerRotationRulesType] = field(default=None, metadata={"description": "A structure that defines the rotation configuration for the secret."}) # fmt: skip + last_rotated_date: Optional[datetime] = field(default=None, metadata={"description": "The most recent date and time that the Secrets Manager rotation process was successfully completed. This value is null if the secret hasn't ever rotated."}) # fmt: skip + last_changed_date: Optional[datetime] = field(default=None, metadata={"description": "The last date and time that this secret was modified in any way."}) # fmt: skip + last_accessed_date: Optional[datetime] = field(default=None, metadata={"description": "The date that the secret was last accessed in the Region. This field is omitted if the secret has never been retrieved in the Region."}) # fmt: skip + deleted_date: Optional[datetime] = field(default=None, metadata={"description": "The date and time the deletion of the secret occurred. Not present on active secrets. The secret can be recovered until the number of days in the recovery window has passed, as specified in the RecoveryWindowInDays parameter of the DeleteSecret operation."}) # fmt: skip + next_rotation_date: Optional[datetime] = field(default=None, metadata={"description": "The next rotation is scheduled to occur on or before this date. If the secret isn't configured for rotation, Secrets Manager returns null."}) # fmt: skip + secret_versions_to_stages: Optional[Dict[str, List[str]]] = field(default=None, metadata={"description": "A list of all of the currently assigned SecretVersionStage staging labels and the SecretVersionId attached to each one. Staging labels are used to keep track of the different versions during the rotation process. A version that does not have any SecretVersionStage is considered deprecated and subject to deletion. Such versions are not included in this list."}) # fmt: skip + owning_service: Optional[str] = field(default=None, metadata={"description": "Returns the name of the service that created the secret."}) # fmt: skip + created_date: Optional[datetime] = field(default=None, metadata={"description": "The date and time when a secret was created."}) # fmt: skip + primary_region: Optional[str] = field(default=None, metadata={"description": "The Region where Secrets Manager originated the secret."}) # fmt: skip + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if kms_key_id := source.get("KmsKeyId"): + builder.dependant_node(from_node=self, clazz=AwsKmsKey, id=AwsKmsKey.normalise_id(kms_key_id)) + + +resources: List[Type[AwsResource]] = [AwsSecretsManagerSecret] diff --git a/plugins/aws/test/collector_test.py b/plugins/aws/test/collector_test.py index 2c88c17f31..3551e0c5a7 100644 --- a/plugins/aws/test/collector_test.py +++ b/plugins/aws/test/collector_test.py @@ -33,8 +33,8 @@ def count_kind(clazz: Type[AwsResource]) -> int: # make sure all threads have been joined assert len(threading.enumerate()) == 1 # ensure the correct number of nodes and edges - assert count_kind(AwsResource) == 210 - assert len(account_collector.graph.edges) == 486 + assert count_kind(AwsResource) == 213 + assert len(account_collector.graph.edges) == 491 assert len(account_collector.graph.deferred_edges) == 2 diff --git a/plugins/aws/test/resources/files/secretsmanager/list-secrets.json b/plugins/aws/test/resources/files/secretsmanager/list-secrets.json new file mode 100644 index 0000000000..f124d9ede4 --- /dev/null +++ b/plugins/aws/test/resources/files/secretsmanager/list-secrets.json @@ -0,0 +1,74 @@ +{ + "SecretList": [ + { + "ARN": "foo", + "Name": "foo", + "Description": "foo", + "KmsKeyId": "32b03fb9", + "RotationEnabled": true, + "RotationLambdaARN": "foo", + "RotationRules": { + "AutomaticallyAfterDays": 123, + "Duration": "foo", + "ScheduleExpression": "foo" + }, + "LastRotatedDate": "2023-12-19T12:46:41Z", + "LastChangedDate": "2023-12-19T12:46:41Z", + "LastAccessedDate": "2023-12-19T12:46:41Z", + "DeletedDate": "2023-12-19T12:46:41Z", + "NextRotationDate": "2023-12-19T12:46:41Z", + "Tags": [ + { + "Key": "foo", + "Value": "foo" + }, + { + "Key": "foo", + "Value": "foo" + }, + { + "Key": "foo", + "Value": "foo" + } + ], + "SecretVersionsToStages": { + "0": [ + "foo", + "foo", + "foo" + ], + "1": [ + "foo", + "foo", + "foo" + ], + "2": [ + "foo", + "foo", + "foo" + ] + }, + "OwningService": "foo", + "CreatedDate": "2023-12-19T12:46:41Z", + "PrimaryRegion": "foo" + }, + { + "ARN":"arn:aws:secretsmanager:us-east-1:test:secret:MyTestDatabaseSecret-a1b2c3", + "Description":"My test database secret", + "LastChangedDate":"2023-12-19T13:50:33.632000+01:00", + "Name":"MyTestDatabaseSecret", + "SecretVersionsToStages":{ + "EXAMPLE2-90ab-cdef-fedc-ba987EXAMPLE":["AWSCURRENT"] + } + }, + { + "ARN":"arn:aws:secretsmanager:us-east-1:test:secret:AnotherDatabaseSecret-d4e5f6", + "Description":"Another secret created for a different database", + "LastChangedDate":"2023-12-19T13:50:33.632000+01:00", + "Name":"AnotherDatabaseSecret", + "SecretVersionsToStages":{ + "EXAMPLE3-90ab-cdef-fedc-ba987EXAMPLE":["AWSCURRENT"] + } + } ], + "NextToken": "foo" +} diff --git a/plugins/aws/test/resources/secretsmanager_test.py b/plugins/aws/test/resources/secretsmanager_test.py new file mode 100644 index 0000000000..14d3bd49e6 --- /dev/null +++ b/plugins/aws/test/resources/secretsmanager_test.py @@ -0,0 +1,7 @@ +from resoto_plugin_aws.resource.secretsmanager import AwsSecretsManagerSecret +from test.resources import round_trip_for + + +def test_notebooks() -> None: + first, builder = round_trip_for(AwsSecretsManagerSecret) + assert len(builder.resources_of(AwsSecretsManagerSecret)) == 3 diff --git a/plugins/aws/tools/aws_model_gen.py b/plugins/aws/tools/aws_model_gen.py index 46c9bad00e..6bb9470364 100644 --- a/plugins/aws/tools/aws_model_gen.py +++ b/plugins/aws/tools/aws_model_gen.py @@ -715,7 +715,7 @@ def default_imports() -> str: ], "ecr": [ # AwsResotoModel("describe-repositories", "repositories", "Repository", prefix="Ecr"), - AwsResotoModel("describe-images", "images", "Image", prefix="Ecr"), + # AwsResotoModel("describe-images", "images", "Image", prefix="Ecr"), ], "eks": [ # AwsResotoModel("list-clusters", "clusters", "Cluster", prefix="Eks", prop_prefix="cluster_"), @@ -997,12 +997,16 @@ def default_imports() -> str: "ssm": [ # AwsResotoModel("describe-instance-information", "InstanceInformationList", "InstanceInformation", prefix="SSM"), ], + "secretsmanager": [ + # AwsResotoModel( "list-secrets", "SecretList", "SecretListEntry", prefix="SecretsManager", name="AwsSecretsManagerSecret" ), + # AwsResotoModel("list-secrets", "SecretList", "SecretVersionStagesType", prefix="SecretsManager"), + ], } if __name__ == "__main__": """print some test data""" - print(json.dumps(create_test_response("ecr", "describe-repositories"), indent=2)) + print(json.dumps(create_test_response("secretsmanager", "list-secrets"), indent=2)) """print the class models""" # print(default_imports()) diff --git a/resotocore/resotocore/report/inspector_service.py b/resotocore/resotocore/report/inspector_service.py index ba3efb4837..5bbad8f0ff 100644 --- a/resotocore/resotocore/report/inspector_service.py +++ b/resotocore/resotocore/report/inspector_service.py @@ -514,7 +514,7 @@ async def start(self) -> None: if not self.cli.dependencies.config.multi_tenant_setup: # TODO: we need a migration path for checks added in existing configs config_ids = {i async for i in self.config_handler.list_config_ids()} - overwrite = False # Only here to simplify development. True until we reach a stable version. + overwrite = True # Only here to simplify development. True until we reach a stable version. for name, js in BenchmarkConfig.from_files().items(): if overwrite or benchmark_id(name) not in config_ids: cid = benchmark_id(name) diff --git a/resotocore/resotocore/static/report/checks/aws/aws_ecr.json b/resotocore/resotocore/static/report/checks/aws/aws_ecr.json new file mode 100644 index 0000000000..eae76260f8 --- /dev/null +++ b/resotocore/resotocore/static/report/checks/aws/aws_ecr.json @@ -0,0 +1,21 @@ +{ + "provider": "aws", + "service": "ecr", + "checks": [ + { + "name": "image_scan_on_push", + "title": "Check if ECR image scan on push is enabled.", + "result_kind": "aws_ecr_repository", + "categories": [], + "risk": "Amazon ECR image scanning helps in identifying software vulnerabilities in your container images. Amazon ECR uses the Common Vulnerabilities and Exposures (CVEs) database from the open-source Clair project and provides a list of scan findings. ", + "severity": "medium", + "detect": { + "resoto": "is(aws_ecr_repository) and image_scan_on_push = false" + }, + "remediation": { + "text": "Enable ECR image scanning and review the scan findings for information about the security of the container images that are being deployed.", + "url": "https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html" + } + } + ] +} diff --git a/resotocore/resotocore/static/report/checks/aws/aws_sagemaker.json b/resotocore/resotocore/static/report/checks/aws/aws_sagemaker.json new file mode 100644 index 0000000000..7df4c78870 --- /dev/null +++ b/resotocore/resotocore/static/report/checks/aws/aws_sagemaker.json @@ -0,0 +1,24 @@ +{ + "provider": "aws", + "service": "sagemaker", + "checks": [ + { + "name": "notebook_root_access", + "title": "Check if Amazon SageMaker Notebook instances have root access disabled", + "result_kind": ["aws_sagemaker_notebook"], + "categories": [ + "security", + "compliance" + ], + "risk": "Users with root access have administrator privileges. Users can access and edit all files on a notebook instance with root access enabled.", + "severity": "medium", + "detect": { + "resoto": "is(aws_sagemaker_notebook) and notebook_root_access==Enabled" + }, + "remediation": { + "text": "Set the RootAccess field to Disabled. You can also disable root access for users when you create or update a notebook instance in the Amazon SageMaker console.", + "url": "https://docs.aws.amazon.com/sagemaker/latest/dg/nbi-root-access.html" + } + } + ] +} From f8d7ab1e076c3f2f1eba2eacdc3d85b3bfbcf6d9 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Tue, 19 Dec 2023 17:34:37 +0100 Subject: [PATCH 04/10] cloudformation: list stack resources --- .../aws/resoto_plugin_aws/resource/base.py | 6 +++++ .../resource/cloudformation.py | 25 +++++++++++++++++-- .../resource/secretsmanager.py | 4 ++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/plugins/aws/resoto_plugin_aws/resource/base.py b/plugins/aws/resoto_plugin_aws/resource/base.py index 260fd7b4e5..acf0c0bf4a 100644 --- a/plugins/aws/resoto_plugin_aws/resource/base.py +++ b/plugins/aws/resoto_plugin_aws/resource/base.py @@ -214,6 +214,8 @@ def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> # In case additional work needs to be done, override this method. for js in json: if instance := cls.from_api(js, builder): + # post process + instance.post_process(builder, js) builder.add_node(instance, js) @classmethod @@ -234,6 +236,10 @@ def called_collect_apis(cls) -> List[AwsApiSpec]: def called_mutator_apis(cls) -> List[AwsApiSpec]: return [] + def post_process(self, builder: GraphBuilder, source: Json) -> None: + # Default behavior: do nothing + pass + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: # Default behavior: add resource to the namespace pass diff --git a/plugins/aws/resoto_plugin_aws/resource/cloudformation.py b/plugins/aws/resoto_plugin_aws/resource/cloudformation.py index c8aac5028c..4d2aa4e44a 100644 --- a/plugins/aws/resoto_plugin_aws/resource/cloudformation.py +++ b/plugins/aws/resoto_plugin_aws/resource/cloudformation.py @@ -7,7 +7,7 @@ from resoto_plugin_aws.aws_client import AwsClient from resoto_plugin_aws.resource.base import AwsResource, AwsApiSpec, GraphBuilder from resoto_plugin_aws.utils import ToDict -from resotolib.baseresources import BaseStack +from resotolib.baseresources import BaseStack, ModelReference from resotolib.graph import ByNodeId, BySearchCriteria, Graph from resotolib.json_bender import Bender, S, Bend, ForallBend, F from resotolib.types import Json @@ -95,6 +95,7 @@ class AwsCloudFormationStack(AwsResource, BaseStack): " updated, or deleted together as a single unit." ) api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "describe-stacks", "Stacks") + reference_kinds: ClassVar[ModelReference] = {"successors": {"default": ["aws_resource"]}} mapping: ClassVar[Dict[str, Bender]] = { "id": S("StackId"), "tags": S("Tags", default=[]) >> ToDict(), @@ -134,6 +135,26 @@ class AwsCloudFormationStack(AwsResource, BaseStack): stack_parent_id: Optional[str] = field(default=None) stack_root_id: Optional[str] = field(default=None) stack_drift_information: Optional[AwsCloudFormationStackDriftInformation] = field(default=None) + _stack_resources: Optional[List[Json]] = None + + def post_process(self, builder: GraphBuilder, source: Json) -> None: + def stack_resources() -> None: + # list all stack resources - we will create edges in connect_in_graph + self._stack_resources = builder.client.list( + service_name, "list-stack-resources", "StackResourceSummaries", StackName=self.name + ) + + builder.submit_work(service_name, stack_resources) + + def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: + if self._stack_resources: + for resource in self._stack_resources: + if (rid := resource.get("PhysicalResourceId")) and (rt := resource.get("ResourceType")): + # we translate the resource type to the internal kind: AWS::IAM::User --> aws_iam_user + # there are a lot of exceptions in AWS, that we need to handle. + kind = rt.replace("::", "_").lower() + # what cloudformation calls "PhysicalResourceId" is the name of the resource + builder.add_edge(self, name=rid, kind=kind) def _modify_tag(self, client: AwsClient, key: str, value: Optional[str], mode: Literal["delete", "update"]) -> bool: tags = dict(self.tags) @@ -202,7 +223,7 @@ def delete_resource(self, client: AwsClient, graph: Graph) -> bool: @classmethod def called_collect_apis(cls) -> List[AwsApiSpec]: - return [cls.api_spec, AwsApiSpec(service_name, "list-stacks")] + return [cls.api_spec, AwsApiSpec(service_name, "list-stacks"), AwsApiSpec(service_name, "list-stack-resources")] @classmethod def called_mutator_apis(cls) -> List[AwsApiSpec]: diff --git a/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py b/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py index 8825549118..345bc01239 100644 --- a/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py +++ b/plugins/aws/resoto_plugin_aws/resource/secretsmanager.py @@ -9,6 +9,8 @@ from resotolib.json_bender import Bender, S, Bend from resotolib.types import Json +service_name = "secretsmanager" + @define(eq=False, slots=False) class AwsSecretsManagerRotationRulesType: @@ -26,7 +28,7 @@ class AwsSecretsManagerRotationRulesType: @define(eq=False, slots=False) class AwsSecretsManagerSecret(AwsResource): kind: ClassVar[str] = "aws_secrets_manager_secret" - api_spec: ClassVar[AwsApiSpec] = AwsApiSpec("secretsmanager", "list-secrets", "SecretList") + api_spec: ClassVar[AwsApiSpec] = AwsApiSpec(service_name, "list-secrets", "SecretList") mapping: ClassVar[Dict[str, Bender]] = { "id": S("Name"), "tags": S("Tags", default=[]) >> ToDict(), From 0c9014d2fdabe584fb776174fb0ba8291b5aaa3a Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 20 Dec 2023 13:26:20 +0100 Subject: [PATCH 05/10] add detect-secrets command --- requirements-all.txt | 1 + requirements-extra.txt | 1 + requirements-test.txt | 1 + requirements.txt | 1 + resotocore/pyproject.toml | 1 + resotocore/resotocore/cli/command.py | 111 ++++++++++++++++++ resotocore/resotocore/model/model.py | 24 ++++ .../tests/resotocore/cli/command_test.py | 25 +++- .../tests/resotocore/hypothesis_extension.py | 2 +- .../tests/resotocore/model/model_test.py | 8 ++ resotocore/tests/resotocore/web/api_test.py | 2 +- 11 files changed, 172 insertions(+), 5 deletions(-) diff --git a/requirements-all.txt b/requirements-all.txt index ad0706b88c..d25f14475f 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -39,6 +39,7 @@ cryptography==41.0.7 deepdiff==6.7.1 defusedxml==0.7.1 deprecated==1.2.14 +detect_secrets==1.4.0 dill==0.3.7 distlib==0.3.7 fastjsonschema==2.19.0 diff --git a/requirements-extra.txt b/requirements-extra.txt index d4c9f789df..2fcfe3887e 100644 --- a/requirements-extra.txt +++ b/requirements-extra.txt @@ -32,6 +32,7 @@ cryptography==41.0.7 deepdiff==6.7.1 defusedxml==0.7.1 deprecated==1.2.14 +detect_secrets==1.4.0 fastjsonschema==2.19.0 filelock==3.13.1 frozendict==2.3.10 diff --git a/requirements-test.txt b/requirements-test.txt index 1ed8df9ec5..35aaa5ec84 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -38,6 +38,7 @@ cryptography==41.0.7 deepdiff==6.7.1 defusedxml==0.7.1 deprecated==1.2.14 +detect_secrets==1.4.0 dill==0.3.7 distlib==0.3.7 fastjsonschema==2.19.0 diff --git a/requirements.txt b/requirements.txt index d9bf2bc8e6..ac9a625826 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,7 @@ cryptography==41.0.7 deepdiff==6.7.1 defusedxml==0.7.1 deprecated==1.2.14 +detect_secrets==1.4.0 fastjsonschema==2.19.0 frozendict==2.3.10 frozenlist==1.4.0 diff --git a/resotocore/pyproject.toml b/resotocore/pyproject.toml index 4df52a5d0c..da48b26731 100644 --- a/resotocore/pyproject.toml +++ b/resotocore/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "aiostream", "cryptography", "deepdiff", + "detect_secrets", "frozendict", "jq", "jsons", diff --git a/resotocore/resotocore/cli/command.py b/resotocore/resotocore/cli/command.py index 388537297f..d604f080d1 100644 --- a/resotocore/resotocore/cli/command.py +++ b/resotocore/resotocore/cli/command.py @@ -37,6 +37,7 @@ FrozenSet, Union, TYPE_CHECKING, + Iterator, ) from urllib.parse import urlparse, urlunparse @@ -51,6 +52,9 @@ from attr import evolve from attrs import define, field from dateutil import parser as date_parser +from detect_secrets.core import scan, plugins +from detect_secrets.core.potential_secret import PotentialSecret +from detect_secrets.settings import configure_settings_from_baseline, default_settings from parsy import Parser, string, ParseError from resotoclient.models import Model as RCModel, Kind as RCKind from resotodatalink import EngineConfig @@ -123,6 +127,7 @@ PropertyPath, TransformKind, AnyKind, + EmptyPath, ) from resotocore.model.resolve_in_graph import NodePath from resotocore.model.typed_model import to_json, to_js, from_js @@ -2428,6 +2433,7 @@ class ListCommand(CLICommand, OutputTransformer): If no prop is defined a predefined list of properties will be shown: + - /info as info - /reported.kind as kind - /reported.id as id - /reported.name as name @@ -2520,6 +2526,7 @@ class ListCommand(CLICommand, OutputTransformer): # This is the list of properties to show in the list command by default default_properties_to_show = [ + (["info"], "info"), (["reported", "kind"], "kind"), (["reported", "id"], "id"), (["reported", "name"], "name"), @@ -6141,6 +6148,109 @@ def parse_duration_or_int(s: str) -> Union[int, timedelta]: ) +class DetectSecretsCommand(CLICommand): + """ + ``` + detect-secrets [--path ] [--with-secrets] + ``` + + The detect-secrets command is able to detect secrets in resources or strings. + + A path can be defined, to not check the whole resource but only a specific property. + This is usually recommended, since a lot of the properties are not relevant for secret detection. + + If the `--with-secrets` flag is provided, the command will filter all incoming resources and pass + only those that contain secrets. + + + ## Parameters + + - `--path` - The property path to check on every incoming element. + - `--with-secrets` - Filter elements to only pass if a secret has been detected. + + ## Examples + + ``` + # Detect secrets anywhere in the node. Also return all resources no matter if secrets are found. + > search is(kubernetes_pod) | detect-secrets + potential_secret=AWS_SECRET_ACCESS_KEY=aeDrhaA3tXjkwIVJ43PHmkCi5, secret_detected=true, secret_type=Secret Keyword, + kind=kubernetes_pod, name=collect, age=14d19h, last_update=90s, cloud=k8s, account=dev, region=jobs + + # Detect secrets anywhere in the node. Only return if a secret is found. + > search is(kubernetes_pod) | detect-secrets --with-secrets + potential_secret=AWS_SECRET_ACCESS_KEY=aeDrhaA3tXjkwIVJ43PHmkCi5, secret_detected=true, secret_type=Secret Keyword, + kind=kubernetes_pod, name=collect, age=14d19h, last_update=90s, cloud=k8s, account=dev, region=jobs + + # Find a secret with a specific path in a resource. Only return if a secret is found. + > search is(kubernetes_pod) | detect-secrets --with-secrets --path pod_spec.containers[*].args[*] + potential_secret=AWS_SECRET_ACCESS_KEY=aeDrhaA3tXjkwIVJ43PHmkCi5, secret_detected=true, secret_type=Secret Keyword, + kind=kubernetes_pod, name=collect, age=14d19h, last_update=90s, cloud=k8s, account=dev, region=jobs + ``` + """ + + @property + def name(self) -> str: + return "detect-secrets" + + def args_info(self) -> ArgsInfo: + return [ArgInfo("--path", expects_value=True, value_hint="property"), ArgInfo("--with-secrets")] + + def info(self) -> str: + return "Detect secrets in resources or strings" + + @lru_cache(maxsize=1) # cache is only used to not configure multiple times + def configure_detect(self) -> None: + with default_settings() as settings: + sj = settings.json() + # adjust the settings: + plugin_settings = defaultdict(dict, {"HexHighEntropyString": {"limit": 3}}) + sj["plugins_used"] = [p | plugin_settings[p["name"]] for p in sj.get("plugins_used", [])] + # make this the default settings + configure_settings_from_baseline(sj) + + def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIAction: + parser = NoExitArgumentParser() + parser.add_argument("--path", type=lambda x: PropertyPath.from_string(ctx.variable_in_section(x))) + parser.add_argument("--with-secrets", action="store_true") + parsed = parser.parse_args(args_parts_unquoted_parser.parse(arg) if arg else []) + + def walk_element(el: JsonElement) -> Iterator[Tuple[str, PotentialSecret]]: + if isinstance(el, dict): + for v in el.values(): + yield from walk_element(v) + elif isinstance(el, list): + for v in el: + yield from walk_element(v) + elif isinstance(el, str) and len(el): + for secret in scan.scan_line(el): + r: str = plugins.initialize.from_secret_type(secret.type).format_scan_result(secret) # type: ignore + if r.startswith("True"): + yield el, secret + + async def detect_secrets_in(content: JsStream) -> JsGen: + self.configure_detect() # make sure all plugins are loaded + async with content.stream() as in_stream: + async for element in in_stream: + path = parsed.path or (PropertyPath.from_list([ctx.section]) if is_node(element) else EmptyPath) + to_check_js = element if path is None else path.value_in(element) + if to_check_js: + found_secrets = False + for secret_string, possible_secret in walk_element(to_check_js): + found_secrets = True + if isinstance(element, dict): + element["info"] = { + "secret_detected": True, + "potential_secret": secret_string, + "secret_type": possible_secret.type, + } + yield element + break + if not found_secrets and not parsed.with_secrets: + yield element + + return CLIFlow(detect_secrets_in) + + def all_commands(d: TenantDependencies) -> List[CLICommand]: commands = [ AggregateCommand(d, "search"), @@ -6151,6 +6261,7 @@ def all_commands(d: TenantDependencies) -> List[CLICommand]: CleanCommand(d, "action"), ConfigsCommand(d, "setup", allowed_in_source_position=True), CountCommand(d, "search"), + DetectSecretsCommand(d, "action"), DbCommand(d, "action", allowed_in_source_position=True), DescendantsPart(d, "search"), DumpCommand(d, "format"), diff --git a/resotocore/resotocore/model/model.py b/resotocore/resotocore/model/model.py index 9d9a95f9df..40dd07cb66 100644 --- a/resotocore/resotocore/model/model.py +++ b/resotocore/resotocore/model/model.py @@ -207,6 +207,30 @@ def same_as(self, other: PropertyPath) -> bool: else: return False + def value_in(self, js: JsonElement) -> JsonElement: + at = len(self.path) + + def at_idx(current: JsonElement, idx: int) -> Optional[Any]: + if at == idx: + return current + path = self.path[idx] + is_array = False + if path and path.endswith("[]"): + path = path[:-2] + is_array = True + if current is None or not isinstance(current, dict) or path not in current: + return None + if is_array: + elem = current[path] + if isinstance(elem, list): + return [at_idx(a, idx + 1) for a in elem] + else: + return None + else: + return at_idx(current[path], idx + 1) + + return at_idx(js, 0) + def __repr__(self) -> str: return self.path_str diff --git a/resotocore/tests/resotocore/cli/command_test.py b/resotocore/tests/resotocore/cli/command_test.py index c86239c277..3b0ad3bdf9 100644 --- a/resotocore/tests/resotocore/cli/command_test.py +++ b/resotocore/tests/resotocore/cli/command_test.py @@ -5,7 +5,7 @@ import sqlite3 from datetime import timedelta from functools import partial -from typing import List, Dict, Optional, Any, Tuple, Type, TypeVar, cast, Callable, Set, Awaitable +from typing import List, Dict, Optional, Any, Tuple, Type, TypeVar, cast, Callable, Set import pytest import yaml @@ -17,15 +17,15 @@ from pytest import fixture from resotocore import version -from resotocore.cli import is_node, JsGen, JsStream, list_sink +from resotocore.cli import is_node, JsStream, list_sink from resotocore.cli.cli import CLIService from resotocore.cli.command import HttpCommand, JqCommand, AggregateCommand, all_commands -from resotocore.dependencies import TenantDependencies from resotocore.cli.model import CLIContext, WorkerCustomCommand, CLI, FilePath from resotocore.cli.tip_of_the_day import generic_tips from resotocore.console_renderer import ConsoleRenderer, ConsoleColorSystem from resotocore.db.graphdb import ArangoGraphDB from resotocore.db.jobdb import JobDb +from resotocore.dependencies import TenantDependencies from resotocore.error import CLIParseError from resotocore.graph_manager.graph_manager import GraphManager from resotocore.ids import InfraAppName, GraphName @@ -1428,3 +1428,22 @@ async def exec(cmd: str) -> List[JsonElement]: # Combine over identifier (which is unique for each entry), filter for identifier==2 --> 1 entry for one timestamp res = await exec(f'timeseries get --name test --start {one_min_ago} --end {in_one_min} --group identifier --filter identifier=="2"') # fmt: skip assert len(res) == 1 + + +@pytest.mark.asyncio +async def test_detect_secrets(cli: CLI) -> None: + async def detect(to_check: JsonElement) -> List[JsonElement]: + res = await cli.execute_cli_command(f"json {json.dumps(to_check)} | detect-secrets --with-secrets", list_sink) + return cast(List[JsonElement], res[0]) + + assert await detect({"foo": 'AWS_SECRET_ACCESS_KEY="aeDrhaA3tXjkwIVJ43PHmkCi5"'}) == [ + { + "foo": 'AWS_SECRET_ACCESS_KEY="aeDrhaA3tXjkwIVJ43PHmkCi5"', + "info": { + "potential_secret": 'AWS_SECRET_ACCESS_KEY="aeDrhaA3tXjkwIVJ43PHmkCi5"', + "secret_detected": True, + "secret_type": "Secret Keyword", + }, + } + ] + assert await detect({"foo": "innocent string"}) == [] diff --git a/resotocore/tests/resotocore/hypothesis_extension.py b/resotocore/tests/resotocore/hypothesis_extension.py index 80bf8647eb..25d2b4ebe5 100644 --- a/resotocore/tests/resotocore/hypothesis_extension.py +++ b/resotocore/tests/resotocore/hypothesis_extension.py @@ -43,7 +43,7 @@ def optional(self, st: SearchStrategy[T]) -> Optional[T]: return self.draw(optional(st)) -any_ws_digits_string = text(alphabet=string.ascii_letters + string.whitespace + string.digits, min_size=0, max_size=10) +any_ws_digits_string = text(alphabet=string.ascii_letters + " " + string.digits, min_size=0, max_size=10) any_string = text(alphabet=string.ascii_letters, min_size=3, max_size=10) kind_gen = sampled_from(["volume", "instance", "load_balancer", "volume_type"]) diff --git a/resotocore/tests/resotocore/model/model_test.py b/resotocore/tests/resotocore/model/model_test.py index 7a7eb77aa8..e869262d94 100644 --- a/resotocore/tests/resotocore/model/model_test.py +++ b/resotocore/tests/resotocore/model/model_test.py @@ -621,6 +621,14 @@ def test_complete_path(person_model: Model) -> None: assert all_props == {"tags": "dictionary[string, string]", "zip": "string"} +def test_property_path_value() -> None: + example = {"test": {"a": {"b": {"c": 1, "d": [{"e": 2}, {"e": 3}], "f": [1, 2, 3, 4]}}}} + assert PropertyPath.from_string("test.a.b.c").value_in(example) == 1 + assert PropertyPath.from_string("test.a.b.d[*].e").value_in(example) == [2, 3] + assert PropertyPath.from_string("test.a.b.f").value_in(example) == [1, 2, 3, 4] + assert PropertyPath.from_string("test.a.b.f[*]").value_in(example) == [1, 2, 3, 4] + + @given(json_object_gen) @settings(max_examples=200, suppress_health_check=list(HealthCheck)) def test_yaml_generation(js: Json) -> None: diff --git a/resotocore/tests/resotocore/web/api_test.py b/resotocore/tests/resotocore/web/api_test.py index 307e8fed00..f9cc2d0eac 100644 --- a/resotocore/tests/resotocore/web/api_test.py +++ b/resotocore/tests/resotocore/web/api_test.py @@ -388,7 +388,7 @@ async def test_cli(core_client: ResotoClient) -> None: # list all cli commands info = AccessJson(await core_client.cli_info()) - assert len(info.commands) == 45 + assert len(info.commands) == 46 @pytest.mark.asyncio From cce498b532f1f155693a8bb0e4cee2aafb6fefbc Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 20 Dec 2023 13:41:33 +0100 Subject: [PATCH 06/10] cleanup imports --- plugins/aws/resoto_plugin_aws/resource/ecr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/aws/resoto_plugin_aws/resource/ecr.py b/plugins/aws/resoto_plugin_aws/resource/ecr.py index ee148bdea6..a15777ad3d 100644 --- a/plugins/aws/resoto_plugin_aws/resource/ecr.py +++ b/plugins/aws/resoto_plugin_aws/resource/ecr.py @@ -1,10 +1,9 @@ -from datetime import datetime from typing import ClassVar, Dict, Optional, List, Type from attrs import define, field from resoto_plugin_aws.resource.base import AwsResource, AwsApiSpec -from resoto_plugin_aws.utils import TagsValue, ToDict +from resoto_plugin_aws.utils import ToDict from resotolib.json_bender import Bender, S, Bend service_name = "ecs" From 4d90ff9d25b2e8e73ef39a9c2c4b651927690c0a Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 20 Dec 2023 13:48:27 +0100 Subject: [PATCH 07/10] fix comment in model gen --- plugins/aws/tools/aws_model_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/aws/tools/aws_model_gen.py b/plugins/aws/tools/aws_model_gen.py index 6bb9470364..862b318eda 100644 --- a/plugins/aws/tools/aws_model_gen.py +++ b/plugins/aws/tools/aws_model_gen.py @@ -5,7 +5,7 @@ import boto3 from attrs import define -from bs4 import BeautifulSoup # pip install beautifulsoup4 lxml +from bs4 import BeautifulSoup # pip install beautifulsoup4 lxml from botocore.model import ServiceModel, StringShape, ListShape, Shape, StructureShape, MapShape from jsons import pascalcase From dd3cb326ce0f160b38ae09514d2b5c34d68b0dae Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 20 Dec 2023 14:25:00 +0100 Subject: [PATCH 08/10] add override and allow to disable plugins --- resotocore/resotocore/cli/command.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resotocore/resotocore/cli/command.py b/resotocore/resotocore/cli/command.py index d604f080d1..805fce1311 100644 --- a/resotocore/resotocore/cli/command.py +++ b/resotocore/resotocore/cli/command.py @@ -6203,8 +6203,9 @@ def configure_detect(self) -> None: with default_settings() as settings: sj = settings.json() # adjust the settings: - plugin_settings = defaultdict(dict, {"HexHighEntropyString": {"limit": 3}}) - sj["plugins_used"] = [p | plugin_settings[p["name"]] for p in sj.get("plugins_used", [])] + override = defaultdict(dict, {"HexHighEntropyString": {"limit": 5}, "Base64HighEntropyString": {"limit": 5}}) + disabled: Set[str] = set() + sj["plugins_used"] = [p | override[p["name"]] for p in sj.get("plugins_used", []) if p["name"] not in disabled] # make this the default settings configure_settings_from_baseline(sj) From e767f7fb9b05c694d13e80c2461ec0db15426cf8 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 20 Dec 2023 15:20:35 +0100 Subject: [PATCH 09/10] fetch user data from ec2 instance --- plugins/aws/resoto_plugin_aws/resource/ec2.py | 20 ++++++++++++++++++- ...ribe-instance-attribute__userData_i_1.json | 6 ++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 plugins/aws/test/resources/files/ec2/describe-instance-attribute__userData_i_1.json diff --git a/plugins/aws/resoto_plugin_aws/resource/ec2.py b/plugins/aws/resoto_plugin_aws/resource/ec2.py index 071f9d9d95..f8234c10b4 100644 --- a/plugins/aws/resoto_plugin_aws/resource/ec2.py +++ b/plugins/aws/resoto_plugin_aws/resource/ec2.py @@ -1,3 +1,4 @@ +import base64 import logging from datetime import datetime from typing import ClassVar, Dict, Optional, List, Type, Any @@ -1341,13 +1342,27 @@ class AwsEc2Instance(EC2Taggable, AwsResource, BaseInstance): instance_ipv6_address: Optional[str] = field(default=None) instance_tpm_support: Optional[str] = field(default=None) instance_maintenance_options: Optional[str] = field(default=None) + instance_user_data: Optional[str] = field(default=None) @classmethod def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def fetch_user_data(instance: AwsEc2Instance) -> None: + if ( + result := builder.client.get( + service_name, + "describe-instance-attribute", + "UserData", + InstanceId=instance.id, + Attribute="userData", + ) + ) and (data := result.get("Value")): + instance.instance_user_data = base64.b64decode(data).decode("utf-8") + for reservation in json: for instance_in in reservation["Instances"]: mapped = bend(cls.mapping, instance_in) instance = AwsEc2Instance.from_json(mapped) + builder.submit_work(service_name, fetch_user_data, instance) builder.add_node(instance, instance_in) @classmethod @@ -1465,7 +1480,10 @@ def delete_resource(self, client: AwsClient, graph: Graph) -> bool: @classmethod def called_mutator_apis(cls) -> List[AwsApiSpec]: - return super().called_mutator_apis() + [AwsApiSpec(service_name, "terminate-instances")] + return super().called_mutator_apis() + [ + AwsApiSpec(service_name, "terminate-instances"), + AwsApiSpec(service_name, "describe-instance-attribute"), + ] # endregion diff --git a/plugins/aws/test/resources/files/ec2/describe-instance-attribute__userData_i_1.json b/plugins/aws/test/resources/files/ec2/describe-instance-attribute__userData_i_1.json new file mode 100644 index 0000000000..b11326a41d --- /dev/null +++ b/plugins/aws/test/resources/files/ec2/describe-instance-attribute__userData_i_1.json @@ -0,0 +1,6 @@ +{ + "InstanceId": "i-0b514e859adc4277e", + "UserData": { + "Value": "dGVzdAo=" + } +} From d7d1241481ea9c4545cb3baefce8552d1089b898 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 20 Dec 2023 16:54:37 +0100 Subject: [PATCH 10/10] allow multiple paths to be checked --- resotocore/resotocore/cli/command.py | 50 +++++++++++++++++----------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/resotocore/resotocore/cli/command.py b/resotocore/resotocore/cli/command.py index 805fce1311..4128690467 100644 --- a/resotocore/resotocore/cli/command.py +++ b/resotocore/resotocore/cli/command.py @@ -6165,7 +6165,7 @@ class DetectSecretsCommand(CLICommand): ## Parameters - - `--path` - The property path to check on every incoming element. + - `--path` - The property path to check on every incoming element. Can be defined multiple times. - `--with-secrets` - Filter elements to only pass if a secret has been detected. ## Examples @@ -6193,7 +6193,10 @@ def name(self) -> str: return "detect-secrets" def args_info(self) -> ArgsInfo: - return [ArgInfo("--path", expects_value=True, value_hint="property"), ArgInfo("--with-secrets")] + return [ + ArgInfo("--path", expects_value=True, value_hint="property", can_occur_multiple_times=True), + ArgInfo("--with-secrets"), + ] def info(self) -> str: return "Detect secrets in resources or strings" @@ -6211,7 +6214,13 @@ def configure_detect(self) -> None: def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIAction: parser = NoExitArgumentParser() - parser.add_argument("--path", type=lambda x: PropertyPath.from_string(ctx.variable_in_section(x))) + parser.add_argument( + "--path", + type=lambda x: PropertyPath.from_string(ctx.variable_in_section(x)), + nargs="*", + default=[], + action="append", + ) parser.add_argument("--with-secrets", action="store_true") parsed = parser.parse_args(args_parts_unquoted_parser.parse(arg) if arg else []) @@ -6232,22 +6241,25 @@ async def detect_secrets_in(content: JsStream) -> JsGen: self.configure_detect() # make sure all plugins are loaded async with content.stream() as in_stream: async for element in in_stream: - path = parsed.path or (PropertyPath.from_list([ctx.section]) if is_node(element) else EmptyPath) - to_check_js = element if path is None else path.value_in(element) - if to_check_js: - found_secrets = False - for secret_string, possible_secret in walk_element(to_check_js): - found_secrets = True - if isinstance(element, dict): - element["info"] = { - "secret_detected": True, - "potential_secret": secret_string, - "secret_type": possible_secret.type, - } - yield element - break - if not found_secrets and not parsed.with_secrets: - yield element + paths = [p for pl in parsed.path for p in pl] + paths = paths or [PropertyPath.from_list([ctx.section]) if is_node(element) else EmptyPath] + found_secrets = False + for path in paths: + if to_check_js := path.value_in(element): + for secret_string, possible_secret in walk_element(to_check_js): + found_secrets = True + if isinstance(element, dict): + element["info"] = { + "secret_detected": True, + "potential_secret": secret_string, + "secret_type": possible_secret.type, + } + yield element + break + if found_secrets: + break # no need to check other paths + if not found_secrets and not parsed.with_secrets: + yield element return CLIFlow(detect_secrets_in)