From d77c65ccb2ea18f748aeeddb2038ffcab0ea1641 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Thu, 17 Oct 2024 13:46:53 +0200 Subject: [PATCH] [aws][feat] Ignore datetime for history by default (#2256) --- fixcore/fixcore/model/graph_access.py | 13 ++++++++----- .../tests/fixcore/model/graph_access_test.py | 9 +++++++++ .../aws/fix_plugin_aws/resource/dynamodb.py | 4 ++-- plugins/aws/fix_plugin_aws/resource/ecs.py | 18 +++++++++--------- plugins/aws/fix_plugin_aws/resource/iam.py | 2 +- plugins/aws/fix_plugin_aws/resource/rds.py | 6 +++--- .../fix_plugin_aws/resource/secretsmanager.py | 2 +- plugins/aws/fix_plugin_aws/resource/ssm.py | 6 +++--- .../fix_plugin_azure/resource/authorization.py | 12 ++++++------ .../fix_plugin_azure/resource/cosmosdb.py | 2 +- 10 files changed, 43 insertions(+), 31 deletions(-) diff --git a/fixcore/fixcore/model/graph_access.py b/fixcore/fixcore/model/graph_access.py index 2eab2ec603..2a4395e449 100644 --- a/fixcore/fixcore/model/graph_access.py +++ b/fixcore/fixcore/model/graph_access.py @@ -334,7 +334,7 @@ def content_hash( def history_hash(js: Json, kind: Kind) -> str: sha256 = hashlib.sha256() - def walk_element(el: JsonElement, el_kind: Kind) -> None: + def walk_element(el: JsonElement, el_kind: Kind, maybe_prop: Optional[Property]) -> None: if el is None: pass elif isinstance(el_kind, ComplexKind): @@ -342,11 +342,14 @@ def walk_element(el: JsonElement, el_kind: Kind) -> None: elif isinstance(el_kind, ArrayKind): if isinstance(el, list): for elem in el: - walk_element(elem, el_kind.inner) + walk_element(elem, el_kind.inner, maybe_prop) elif isinstance(el_kind, DictionaryKind): if isinstance(el, dict): for _, v in sorted(el.items()): - walk_element(v, el_kind.value_kind) + walk_element(v, el_kind.value_kind, maybe_prop) + elif isinstance(el_kind, (DateKind, DateTimeKind)): # default: ignore, opt-in to keep + if maybe_prop and maybe_prop.meta_get("keep_history", bool, False): + sha256.update(str(el).encode("utf-8")) elif isinstance(el_kind, SimpleKind): sha256.update(str(el).encode("utf-8")) @@ -358,12 +361,12 @@ def walk_complex(el: JsonElement, el_kind: ComplexKind) -> None: and (prop.name not in PropsToIgnoreForHistory) and (prop_val := el.get(prop.name)) ): - walk_element(prop_val, prop_kind) + walk_element(prop_val, prop_kind, prop) if not el_kind.metadata.get("ignore_history"): # if defined on type, do not walk the hierarchy for base in el_kind.resolved_bases().values(): walk_complex(el, base) - walk_element(js, kind) + walk_element(js, kind, None) return sha256.hexdigest()[0:8] @staticmethod diff --git a/fixcore/tests/fixcore/model/graph_access_test.py b/fixcore/tests/fixcore/model/graph_access_test.py index 74a4e836fc..b30e1c617f 100644 --- a/fixcore/tests/fixcore/model/graph_access_test.py +++ b/fixcore/tests/fixcore/model/graph_access_test.py @@ -90,6 +90,15 @@ def test_content_hash() -> None: assert sha1 == sha2 +def test_history_hash(person_model: Model) -> None: + address = person_model["Address"] + a = {"id": "a1", "zip": "s1", "city": "c1", "mtime": "2021-06-18T10:31:34Z"} + b = {"id": "a1", "zip": "s1", "city": "c1", "mtime": "2022-06-18T10:31:34Z"} + c = {"id": "a1", "zip": "s2", "city": "c2", "mtime": "2022-06-18T10:31:34Z"} + assert GraphBuilder.history_hash(a, address) == GraphBuilder.history_hash(b, address) + assert GraphBuilder.history_hash(a, address) != GraphBuilder.history_hash(c, address) + + def test_root(graph_access: GraphAccess) -> None: assert graph_access.root() == "1" diff --git a/plugins/aws/fix_plugin_aws/resource/dynamodb.py b/plugins/aws/fix_plugin_aws/resource/dynamodb.py index a06a5355b2..c068aaf921 100644 --- a/plugins/aws/fix_plugin_aws/resource/dynamodb.py +++ b/plugins/aws/fix_plugin_aws/resource/dynamodb.py @@ -340,8 +340,8 @@ class AwsDynamoDbPointInTimeRecovery: "latest_restorable_date_time": S("LatestRestorableDateTime"), } status: Optional[str] = field(default=None, metadata={"description": "The current state of point in time recovery: ENABLED - Point in time recovery is enabled. DISABLED - Point in time recovery is disabled."}) # fmt: skip - earliest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Specifies the earliest point in time you can restore your table to. You can restore your table to any point in time during the last 35 days."}) # fmt: skip - latest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "LatestRestorableDateTime is typically 5 minutes before the current time."}) # fmt: skip + earliest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"description": "Specifies the earliest point in time you can restore your table to. You can restore your table to any point in time during the last 35 days."}) # fmt: skip + latest_restorable_date_time: Optional[datetime] = field(default=None, metadata={"description": "LatestRestorableDateTime is typically 5 minutes before the current time."}) # fmt: skip @define(eq=False, slots=False) diff --git a/plugins/aws/fix_plugin_aws/resource/ecs.py b/plugins/aws/fix_plugin_aws/resource/ecs.py index 933181f81b..7570e20cb3 100644 --- a/plugins/aws/fix_plugin_aws/resource/ecs.py +++ b/plugins/aws/fix_plugin_aws/resource/ecs.py @@ -488,7 +488,7 @@ class AwsEcsTask(EcsTaggable, AwsResource): task_capacity_provider_name: Optional[str] = field(default=None) task_cluster_arn: Optional[str] = field(default=None) task_connectivity: Optional[str] = field(default=None, metadata=dict(ignore_history=True)) - task_connectivity_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) + task_connectivity_at: Optional[datetime] = field(default=None) task_container_instance_arn: Optional[str] = field(default=None) task_containers: List[AwsEcsContainer] = field(factory=list) task_cpu: Optional[str] = field(default=None) @@ -504,14 +504,14 @@ class AwsEcsTask(EcsTaggable, AwsResource): task_overrides: Optional[AwsEcsTaskOverride] = field(default=None) task_platform_version: Optional[str] = field(default=None) task_platform_family: Optional[str] = field(default=None) - task_pull_started_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) - task_pull_stopped_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) - task_started_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) - task_started_by: Optional[str] = field(default=None, metadata=dict(ignore_history=True)) - task_stop_code: Optional[str] = field(default=None, metadata=dict(ignore_history=True)) - task_stopped_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) - task_stopped_reason: Optional[str] = field(default=None, metadata=dict(ignore_history=True)) - task_stopping_at: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) + task_pull_started_at: Optional[datetime] = field(default=None) + task_pull_stopped_at: Optional[datetime] = field(default=None) + task_started_at: Optional[datetime] = field(default=None) + task_started_by: Optional[str] = field(default=None) + task_stop_code: Optional[str] = field(default=None) + task_stopped_at: Optional[datetime] = field(default=None) + task_stopped_reason: Optional[str] = field(default=None) + task_stopping_at: Optional[datetime] = field(default=None) task_definition_arn: Optional[str] = field(default=None) task_version: Optional[int] = field(default=None) task_ephemeral_storage: Optional[int] = field(default=None) diff --git a/plugins/aws/fix_plugin_aws/resource/iam.py b/plugins/aws/fix_plugin_aws/resource/iam.py index f26ddeebf5..bc6b65db99 100644 --- a/plugins/aws/fix_plugin_aws/resource/iam.py +++ b/plugins/aws/fix_plugin_aws/resource/iam.py @@ -659,7 +659,7 @@ class AwsIamUser(AwsResource, BaseUser, BaseIamPrincipal): user_policies: List[AwsIamPolicyDetail] = field(factory=list) user_permissions_boundary: Optional[AwsIamAttachedPermissionsBoundary] = field(default=None) password_enabled: Optional[bool] = field(default=None) - password_last_used: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) + password_last_used: Optional[datetime] = field(default=None) password_last_changed: Optional[datetime] = field(default=None) password_next_rotation: Optional[datetime] = field(default=None) mfa_active: Optional[bool] = field(default=None) diff --git a/plugins/aws/fix_plugin_aws/resource/rds.py b/plugins/aws/fix_plugin_aws/resource/rds.py index f1d4e11b40..c1427d463a 100644 --- a/plugins/aws/fix_plugin_aws/resource/rds.py +++ b/plugins/aws/fix_plugin_aws/resource/rds.py @@ -464,7 +464,7 @@ class AwsRdsInstance(RdsTaggable, AwsResource, BaseDatabase): rds_db_subnet_group: Optional[AwsRdsDBSubnetGroup] = field(default=None) rds_preferred_maintenance_window: Optional[str] = field(default=None) rds_pending_modified_values: Optional[AwsRdsPendingModifiedValues] = field(default=None) - rds_latest_restorable_time: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) + rds_latest_restorable_time: Optional[datetime] = field(default=None) rds_multi_az: Optional[bool] = field(default=None) rds_auto_minor_version_upgrade: Optional[bool] = field(default=None) rds_read_replica_source_db_instance_identifier: Optional[str] = field(default=None) @@ -973,12 +973,12 @@ class AwsRdsCluster(RdsTaggable, AwsResource, BaseDatabase): rds_db_cluster_parameter_group: Optional[str] = field(default=None) rds_db_subnet_group_name: Optional[str] = field(default=None) rds_automatic_restart_time: Optional[datetime] = field(default=None) - rds_earliest_restorable_time: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) + rds_earliest_restorable_time: Optional[datetime] = field(default=None) rds_endpoint: Optional[str] = field(default=None) rds_reader_endpoint: Optional[str] = field(default=None) rds_custom_endpoints: List[str] = field(factory=list) rds_multi_az: Optional[bool] = field(default=None) - rds_latest_restorable_time: Optional[datetime] = field(default=None, metadata=dict(ignore_history=True)) + rds_latest_restorable_time: Optional[datetime] = field(default=None) rds_port: Optional[int] = field(default=None) rds_master_username: Optional[str] = field(default=None) rds_db_cluster_option_group_memberships: List[AwsRdsDBClusterOptionGroupStatus] = field(factory=list) diff --git a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py index e79f52a172..fd23e7f178 100644 --- a/plugins/aws/fix_plugin_aws/resource/secretsmanager.py +++ b/plugins/aws/fix_plugin_aws/resource/secretsmanager.py @@ -67,7 +67,7 @@ class AwsSecretsManagerSecret(HasResourcePolicy, AwsResource): rotation_rules: Optional[AwsSecretsManagerRotationRulesType] = field(default=None, metadata={"description": "A structure that defines the rotation configuration for the secret."}) # fmt: skip last_rotated_date: Optional[datetime] = field(default=None, metadata={"description": "The most recent date and time that the Secrets Manager rotation process was successfully completed. This value is null if the secret hasn't ever rotated."}) # fmt: skip last_changed_date: Optional[datetime] = field(default=None, metadata={"description": "The last date and time that this secret was modified in any way."}) # fmt: skip - last_accessed_date: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The date that the secret was last accessed in the Region. This field is omitted if the secret has never been retrieved in the Region."}) # fmt: skip + last_accessed_date: Optional[datetime] = field(default=None, metadata={"description": "The date that the secret was last accessed in the Region. This field is omitted if the secret has never been retrieved in the Region."}) # fmt: skip deleted_date: Optional[datetime] = field(default=None, metadata={"description": "The date and time the deletion of the secret occurred. Not present on active secrets. The secret can be recovered until the number of days in the recovery window has passed, as specified in the RecoveryWindowInDays parameter of the DeleteSecret operation."}) # fmt: skip next_rotation_date: Optional[datetime] = field(default=None, metadata={"description": "The next rotation is scheduled to occur on or before this date. If the secret isn't configured for rotation, Secrets Manager returns null."}) # fmt: skip secret_versions_to_stages: Optional[Dict[str, List[str]]] = field(default=None, metadata={"description": "A list of all of the currently assigned SecretVersionStage staging labels and the SecretVersionId attached to each one. Staging labels are used to keep track of the different versions during the rotation process. A version that does not have any SecretVersionStage is considered deprecated and subject to deletion. Such versions are not included in this list."}) # fmt: skip diff --git a/plugins/aws/fix_plugin_aws/resource/ssm.py b/plugins/aws/fix_plugin_aws/resource/ssm.py index dd823f9575..dd7fcf8498 100644 --- a/plugins/aws/fix_plugin_aws/resource/ssm.py +++ b/plugins/aws/fix_plugin_aws/resource/ssm.py @@ -71,7 +71,7 @@ class AwsSSMInstance(AwsResource): } instance_id: Optional[str] = field(default=None, metadata={"description": "The managed node ID."}) # fmt: skip ping_status: Optional[str] = field(default=None, metadata={"description": "Connection status of SSM Agent. The status Inactive has been deprecated and is no longer in use."}) # fmt: skip - last_ping: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The date and time when the agent last pinged the Systems Manager service."}) # fmt: skip + last_ping: Optional[datetime] = field(default=None, metadata={"description": "The date and time when the agent last pinged the Systems Manager service."}) # fmt: skip agent_version: Optional[str] = field(default=None, metadata={"description": "The version of SSM Agent running on your Linux managed node."}) # fmt: skip is_latest_version: Optional[bool] = field(default=None, metadata={"description": "Indicates whether the latest version of SSM Agent is running on your Linux managed node. This field doesn't indicate whether or not the latest version is installed on Windows managed nodes, because some older versions of Windows Server use the EC2Config service to process Systems Manager requests."}) # fmt: skip platform_type: Optional[str] = field(default=None, metadata={"description": "The operating system platform type."}) # fmt: skip @@ -84,8 +84,8 @@ class AwsSSMInstance(AwsResource): ip_address: Optional[str] = field(default=None, metadata={"description": "The IP address of the managed node."}) # fmt: skip computer_name: Optional[str] = field(default=None, metadata={"description": "The fully qualified host name of the managed node."}) # fmt: skip association_status: Optional[str] = field(default=None, metadata={"description": "The status of the association."}) # fmt: skip - last_association_execution_date: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The date the association was last run."}) # fmt: skip - last_successful_association_execution_date: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "The last date the association was successfully run."}) # fmt: skip + last_association_execution_date: Optional[datetime] = field(default=None, metadata={"description": "The date the association was last run."}) # fmt: skip + last_successful_association_execution_date: Optional[datetime] = field(default=None, metadata={"description": "The last date the association was successfully run."}) # fmt: skip association_overview: Optional[AwsSSMInstanceAggregatedAssociationOverview] = field(default=None, metadata={"description": "Information about the association."}) # fmt: skip source_id: Optional[str] = field(default=None, metadata={"description": "The ID of the source resource. For IoT Greengrass devices, SourceId is the Thing name."}) # fmt: skip source_type: Optional[str] = field(default=None, metadata={"description": "The type of the source resource. For IoT Greengrass devices, SourceType is AWS::IoT::Thing."}) # fmt: skip diff --git a/plugins/azure/fix_plugin_azure/resource/authorization.py b/plugins/azure/fix_plugin_azure/resource/authorization.py index 30fdd8f095..0f4c2fd12f 100644 --- a/plugins/azure/fix_plugin_azure/resource/authorization.py +++ b/plugins/azure/fix_plugin_azure/resource/authorization.py @@ -102,7 +102,7 @@ class AzureAuthorizationDenyAssignment(MicrosoftResource): condition: Optional[str] = field(default=None, metadata={'description': 'The conditions on the deny assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase foo_storage_container '}) # fmt: skip condition_version: Optional[str] = field(default=None, metadata={"description": "Version of the condition."}) created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'}) # fmt: skip - created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"}) # fmt: skip + created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"}) # fmt: skip deny_assignment_name: Optional[str] = field(default=None, metadata={'description': 'The display name of the deny assignment.'}) # fmt: skip description: Optional[str] = field(default=None, metadata={'description': 'The description of the deny assignment.'}) # fmt: skip do_not_apply_to_child_scopes: Optional[bool] = field(default=None, metadata={'description': 'Determines if the deny assignment applies to child scopes. Default value is false.'}) # fmt: skip @@ -112,7 +112,7 @@ class AzureAuthorizationDenyAssignment(MicrosoftResource): principals: Optional[List[AzurePrincipal]] = field(default=None, metadata={'description': 'Array of principals to which the deny assignment applies.'}) # fmt: skip scope: Optional[str] = field(default=None, metadata={"description": "The deny assignment scope."}) updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'}) # fmt: skip - updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"}) # fmt: skip + updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"}) # fmt: skip @define(eq=False, slots=False) @@ -179,7 +179,7 @@ class AzureAuthorizationRoleAssignment(MicrosoftResource): condition: Optional[str] = field(default=None, metadata={'description': 'The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase foo_storage_container '}) # fmt: skip condition_version: Optional[str] = field(default=None, metadata={'description': 'Version of the condition. Currently the only accepted value is 2.0 '}) # fmt: skip created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'}) # fmt: skip - created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"}) # fmt: skip + created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"}) # fmt: skip delegated_managed_identity_resource_id: Optional[str] = field(default=None, metadata={'description': 'Id of the delegated managed identity resource'}) # fmt: skip description: Optional[str] = field(default=None, metadata={"description": "Description of role assignment"}) principal_id: Optional[str] = field(default=None, metadata={"description": "The principal ID."}) @@ -187,7 +187,7 @@ class AzureAuthorizationRoleAssignment(MicrosoftResource): role_definition_id: Optional[str] = field(default=None, metadata={"description": "The role definition ID."}) scope: Optional[str] = field(default=None, metadata={"description": "The role assignment scope."}) updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'}) # fmt: skip - updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"}) # fmt: skip + updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"}) # fmt: skip def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: # role definition @@ -259,12 +259,12 @@ class AzureAuthorizationRoleDefinition(MicrosoftResource, BaseRole, PhantomBaseR } assignable_scopes: Optional[List[str]] = field(default=None, metadata={'description': 'Role definition assignable scopes.'}) # fmt: skip created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'}) # fmt: skip - created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"}) # fmt: skip + created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"}) # fmt: skip description: Optional[str] = field(default=None, metadata={"description": "The role definition description."}) azure_role_permissions: Optional[List[AzurePermission]] = field(default=None, metadata={'description': 'Role definition permissions.'}) # fmt: skip role_name: Optional[str] = field(default=None, metadata={"description": "The role name."}) updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'}) # fmt: skip - updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"}) # fmt: skip + updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"}) # fmt: skip @define(eq=False, slots=False) diff --git a/plugins/azure/fix_plugin_azure/resource/cosmosdb.py b/plugins/azure/fix_plugin_azure/resource/cosmosdb.py index 35e4de2a4d..99520734a1 100644 --- a/plugins/azure/fix_plugin_azure/resource/cosmosdb.py +++ b/plugins/azure/fix_plugin_azure/resource/cosmosdb.py @@ -1608,7 +1608,7 @@ class AzureCosmosDBRestorableAccount(CosmosDBLocationSetter, MicrosoftResource): api_type: Optional[str] = field(default=None, metadata={'description': 'Enum to indicate the API type of the restorable database account.'}) # fmt: skip creation_time: Optional[datetime] = field(default=None, metadata={'description': 'The creation time of the restorable database account (ISO-8601 format).'}) # fmt: skip deletion_time: Optional[datetime] = field(default=None, metadata={'description': 'The time at which the restorable database account has been deleted (ISO-8601 format).'}) # fmt: skip - oldest_restorable_time: Optional[datetime] = field(default=None, metadata={"ignore_history": True, 'description': 'The least recent time at which the database account can be restored to (ISO-8601 format).'}) # fmt: skip + oldest_restorable_time: Optional[datetime] = field(default=None, metadata={'description': 'The least recent time at which the database account can be restored to (ISO-8601 format).'}) # fmt: skip restorable_locations: Optional[List[AzureRestorableLocationResource]] = field(default=None, metadata={'description': 'List of regions where the of the database account can be restored from.'}) # fmt: skip def _collect_items(