From c28252a1a3ba41869b8d834c279aa1037ef80115 Mon Sep 17 00:00:00 2001 From: modernisation-platform-ci Date: Mon, 23 Sep 2024 04:53:39 +0000 Subject: [PATCH] Updates from GitHub Actions Format Code workflow --- terraform/environments/apex/ecs.tf | 30 +- terraform/environments/apex/locals.tf | 2 +- .../environments/ccms-ebs/ccms-lambda.tf | 4 +- .../audit_external_table_definition.json | 349 ++++++++-------- ...interaction_checksum_inbound_settings.json | 2 +- ...action_checksum_inbound_table_mapping.json | 64 +-- ...nteraction_checksum_outbound_settings.json | 2 +- ...ction_checksum_outbound_table_mapping.json | 26 +- .../audited_interaction_inbound_settings.json | 69 ++-- ...ted_interaction_inbound_table_mapping.json | 76 ++-- ...audited_interaction_outbound_settings.json | 69 ++-- ...business_interaction_inbound_settings.json | 2 +- ...ess_interaction_inbound_table_mapping.json | 113 +++-- ...usiness_interaction_outbound_settings.json | 2 +- .../files/user_external_table_definition.json | 389 +++++++++--------- .../files/user_inbound_settings.json | 108 ++--- .../files/user_inbound_table_mapping.json | 50 +-- .../files/user_outbound_settings.json | 108 ++--- .../files/user_outbound_table_mapping.json | 50 +-- terraform/environments/delius-core/locals.tf | 10 +- .../delius-core/locals_preproduction.tf | 4 +- .../environments/delius-core/locals_stage.tf | 4 +- .../environments/delius-core/locals_test.tf | 2 +- .../components/dms/cloudwatch-alarms.tf | 54 +-- .../components/dms/dms_db_source_endpoints.tf | 68 +-- .../components/dms/dms_db_target_endpoints.tf | 72 ++-- .../modules/components/dms/dms_iam.tf | 48 +-- .../components/dms/dms_inbound_tasks.tf | 14 +- .../components/dms/dms_outbound_tasks.tf | 28 +- .../modules/components/dms/dms_s3.tf | 54 +-- .../components/dms/dms_s3_source_endpoints.tf | 36 +- .../components/dms/dms_s3_target_endpoints.tf | 44 +- .../modules/components/dms/locals.tf | 28 +- .../modules/components/dms/oracle_wallet.tf | 4 +- .../delius-core/modules/components/dms/sg.tf | 12 +- .../modules/delius_environment/dms.tf | 2 +- .../modules/delius_environment/ldap_ecs.tf | 4 +- .../modules/delius_environment/variables.tf | 2 +- .../helpers/delius_microservice/variables.tf | 4 +- .../modules/ec2/variables.tf | 10 +- .../modules/pipeline_trigger/variables.tf | 4 +- .../digital-prison-reporting/sg.tf | 2 +- .../electronic-monitoring-data/s3.tf | 40 +- terraform/environments/wardship/dns_ssl.tf | 36 +- .../environments/xhibit-portal/shield.tf | 2 +- 45 files changed, 1048 insertions(+), 1055 deletions(-) diff --git a/terraform/environments/apex/ecs.tf b/terraform/environments/apex/ecs.tf index a1f19d8d699..a0a2421b42c 100644 --- a/terraform/environments/apex/ecs.tf +++ b/terraform/environments/apex/ecs.tf @@ -19,20 +19,20 @@ module "apex-ecs" { ec2_min_size = local.application_data.accounts[local.environment].ec2_min_size # task_definition_volume = local.application_data.accounts[local.environment].task_definition_volume # network_mode = local.application_data.accounts[local.environment].network_mode - server_port = local.application_data.accounts[local.environment].server_port - app_count = local.application_data.accounts[local.environment].app_count - ec2_ingress_rules = local.ec2_ingress_rules - ec2_egress_rules = local.ec2_egress_rules - lb_tg_arn = module.alb.target_group_arn - tags_common = local.tags - appscaling_min_capacity = local.application_data.accounts[local.environment].appscaling_min_capacity - appscaling_max_capacity = local.application_data.accounts[local.environment].appscaling_max_capacity - ecs_scaling_cpu_threshold = local.application_data.accounts[local.environment].ecs_scaling_cpu_threshold - ecs_scaling_mem_threshold = local.application_data.accounts[local.environment].ecs_scaling_mem_threshold - ecs_target_capacity = local.ecs_target_capacity - ec2_instance_warmup_period = local.application_data.accounts[local.environment].ec2_instance_warmup_period - log_group_kms_key = aws_kms_key.cloudwatch_logs_key.arn - environment = local.environment - database_admin_password_arn = "arn:aws:ssm:${local.application_data.accounts[local.environment].region}:${local.env_account_id}:parameter/${local.app_db_password_name}" + server_port = local.application_data.accounts[local.environment].server_port + app_count = local.application_data.accounts[local.environment].app_count + ec2_ingress_rules = local.ec2_ingress_rules + ec2_egress_rules = local.ec2_egress_rules + lb_tg_arn = module.alb.target_group_arn + tags_common = local.tags + appscaling_min_capacity = local.application_data.accounts[local.environment].appscaling_min_capacity + appscaling_max_capacity = local.application_data.accounts[local.environment].appscaling_max_capacity + ecs_scaling_cpu_threshold = local.application_data.accounts[local.environment].ecs_scaling_cpu_threshold + ecs_scaling_mem_threshold = local.application_data.accounts[local.environment].ecs_scaling_mem_threshold + ecs_target_capacity = local.ecs_target_capacity + ec2_instance_warmup_period = local.application_data.accounts[local.environment].ec2_instance_warmup_period + log_group_kms_key = aws_kms_key.cloudwatch_logs_key.arn + environment = local.environment + database_admin_password_arn = "arn:aws:ssm:${local.application_data.accounts[local.environment].region}:${local.env_account_id}:parameter/${local.app_db_password_name}" } \ No newline at end of file diff --git a/terraform/environments/apex/locals.tf b/terraform/environments/apex/locals.tf index 3818a632872..2b273d00872 100644 --- a/terraform/environments/apex/locals.tf +++ b/terraform/environments/apex/locals.tf @@ -71,7 +71,7 @@ locals { app_db_url = "${aws_route53_record.apex-db.fqdn}:1521:APEX" app_debug_enabled = local.application_data.accounts[local.environment].app_debug_enabled # Note that the following secret is created manually on Parameter Store - db_secret_arn = "arn:aws:ssm:${local.application_data.accounts[local.environment].region}:${local.env_account_id}:parameter/${local.app_db_password_name}" + db_secret_arn = "arn:aws:ssm:${local.application_data.accounts[local.environment].region}:${local.env_account_id}:parameter/${local.app_db_password_name}" }) env_account_id = local.environment_management.account_ids[terraform.workspace] diff --git a/terraform/environments/ccms-ebs/ccms-lambda.tf b/terraform/environments/ccms-ebs/ccms-lambda.tf index f5442b1d415..4e6b653ae18 100644 --- a/terraform/environments/ccms-ebs/ccms-lambda.tf +++ b/terraform/environments/ccms-ebs/ccms-lambda.tf @@ -20,9 +20,9 @@ resource "aws_lambda_layer_version" "lambda_layer" { # SG for Lambda resource "aws_security_group" "lambda_security_group" { - name = "${local.application_name}-${local.environment}-lambda-sg" + name = "${local.application_name}-${local.environment}-lambda-sg" description = "SG traffic control for Payment Load Lambda" - vpc_id = data.aws_vpc.shared.id + vpc_id = data.aws_vpc.shared.id ingress { from_port = 1521 diff --git a/terraform/environments/delius-core/files/audit_external_table_definition.json b/terraform/environments/delius-core/files/audit_external_table_definition.json index 4121f009e8e..0244495a3c5 100644 --- a/terraform/environments/delius-core/files/audit_external_table_definition.json +++ b/terraform/environments/delius-core/files/audit_external_table_definition.json @@ -1,176 +1,175 @@ { - "TableCount": 3, - "Tables": [ - { - "TableName": "AUDITED_INTERACTION", - "TablePath": "DELIUS_APP_SCHEMA/AUDITED_INTERACTION/", - "TableOwner": "DELIUS_APP_SCHEMA", - "TableColumns": [ - { - "ColumnName": "DMS_CDC_TIMESTAMP", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", - "ColumnNullable": "false", - "ColumnScale": "6" - }, - { - "ColumnName": "DATE_TIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "false" - }, - { - "ColumnName": "OUTCOME", - "ColumnType": "STRING", - "ColumnLength": "1", - "ColumnNullable": "false" - }, - { - "ColumnName": "INTERACTION_PARAMETERS", - "ColumnType": "STRING", - "ColumnLength": "4000", - "ColumnNullable": "true" - }, - { - "ColumnName": "USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "BUSINESS_INTERACTION_ID", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "SPG_USERNAME", - "ColumnType": "STRING", - "ColumnLength": "80", - "ColumnNullable": "true" - }, - { - "ColumnName": "CLIENT_DB", - "ColumnType": "STRING", - "ColumnLength": "20", - "ColumnNullable": "true" - } - ], - "TableColumnsTotal": "8" - }, - { - "TableName": "BUSINESS_INTERACTION", - "TablePath": "DELIUS_APP_SCHEMA/BUSINESS_INTERACTION/", - "TableOwner": "DELIUS_APP_SCHEMA", - "TableColumns": [ - { - "ColumnName": "DMS_CDC_TIMESTAMP", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", - "ColumnNullable": "false", - "ColumnScale": "6" - }, - { - "ColumnName": "BUSINESS_INTERACTION_ID", - "ColumnType": "INT8", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "BUSINESS_INTERACTION_CODE", - "ColumnType": "STRING", - "ColumnLength": "20", - "ColumnNullable": "false" - }, - { - "ColumnName": "DESCRIPTION", - "ColumnType": "STRING", - "ColumnLength": "500", - "ColumnNullable": "false" - }, - { - "ColumnName": "ENABLED_DATE", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "true" - }, - { - "ColumnName": "AUDIT_INTERACTION_PARAMETER", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "true" - }, - { - "ColumnName": "ROW_VERSION", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "CLIENT_DB", - "ColumnType": "STRING", - "ColumnLength": "20", - "ColumnNullable": "false", - "ColumnIsPk": "true" - } - ], - "TableColumnsTotal": "8" - }, - { - "TableName": "AUDITED_INTERACTION_CHECKSUM", - "TablePath": "DELIUS_AUDIT_DMS_POOL/AUDITED_INTERACTION_CHECKSUM/", - "TableOwner": "DELIUS_AUDIT_DMS_POOL", - "TableColumns": [ - { - "ColumnName": "DMS_CDC_TIMESTAMP", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", - "ColumnNullable": "false", - "ColumnScale": "6" - }, - { - "ColumnName": "CLIENT_DB", - "ColumnType": "STRING", - "ColumnLength": "20", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "START_DATE_TIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "RESETLOGS", - "ColumnType": "STRING", - "ColumnLength": "1", - "ColumnNullable": "true" - }, - { - "ColumnName": "END_DATE_TIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "ROW_COUNT", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "DATA_CHECKSUM", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "CHECKSUM_VALIDATED", - "ColumnType": "STRING", - "ColumnLength": "1", - "ColumnNullable": "true" - } - ], - "TableColumnsTotal": "8" - } - ] - } - \ No newline at end of file + "TableCount": 3, + "Tables": [ + { + "TableName": "AUDITED_INTERACTION", + "TablePath": "DELIUS_APP_SCHEMA/AUDITED_INTERACTION/", + "TableOwner": "DELIUS_APP_SCHEMA", + "TableColumns": [ + { + "ColumnName": "DMS_CDC_TIMESTAMP", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", + "ColumnNullable": "false", + "ColumnScale": "6" + }, + { + "ColumnName": "DATE_TIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "false" + }, + { + "ColumnName": "OUTCOME", + "ColumnType": "STRING", + "ColumnLength": "1", + "ColumnNullable": "false" + }, + { + "ColumnName": "INTERACTION_PARAMETERS", + "ColumnType": "STRING", + "ColumnLength": "4000", + "ColumnNullable": "true" + }, + { + "ColumnName": "USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "BUSINESS_INTERACTION_ID", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "SPG_USERNAME", + "ColumnType": "STRING", + "ColumnLength": "80", + "ColumnNullable": "true" + }, + { + "ColumnName": "CLIENT_DB", + "ColumnType": "STRING", + "ColumnLength": "20", + "ColumnNullable": "true" + } + ], + "TableColumnsTotal": "8" + }, + { + "TableName": "BUSINESS_INTERACTION", + "TablePath": "DELIUS_APP_SCHEMA/BUSINESS_INTERACTION/", + "TableOwner": "DELIUS_APP_SCHEMA", + "TableColumns": [ + { + "ColumnName": "DMS_CDC_TIMESTAMP", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", + "ColumnNullable": "false", + "ColumnScale": "6" + }, + { + "ColumnName": "BUSINESS_INTERACTION_ID", + "ColumnType": "INT8", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "BUSINESS_INTERACTION_CODE", + "ColumnType": "STRING", + "ColumnLength": "20", + "ColumnNullable": "false" + }, + { + "ColumnName": "DESCRIPTION", + "ColumnType": "STRING", + "ColumnLength": "500", + "ColumnNullable": "false" + }, + { + "ColumnName": "ENABLED_DATE", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "true" + }, + { + "ColumnName": "AUDIT_INTERACTION_PARAMETER", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "true" + }, + { + "ColumnName": "ROW_VERSION", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "CLIENT_DB", + "ColumnType": "STRING", + "ColumnLength": "20", + "ColumnNullable": "false", + "ColumnIsPk": "true" + } + ], + "TableColumnsTotal": "8" + }, + { + "TableName": "AUDITED_INTERACTION_CHECKSUM", + "TablePath": "DELIUS_AUDIT_DMS_POOL/AUDITED_INTERACTION_CHECKSUM/", + "TableOwner": "DELIUS_AUDIT_DMS_POOL", + "TableColumns": [ + { + "ColumnName": "DMS_CDC_TIMESTAMP", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", + "ColumnNullable": "false", + "ColumnScale": "6" + }, + { + "ColumnName": "CLIENT_DB", + "ColumnType": "STRING", + "ColumnLength": "20", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "START_DATE_TIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "RESETLOGS", + "ColumnType": "STRING", + "ColumnLength": "1", + "ColumnNullable": "true" + }, + { + "ColumnName": "END_DATE_TIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "ROW_COUNT", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "DATA_CHECKSUM", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "CHECKSUM_VALIDATED", + "ColumnType": "STRING", + "ColumnLength": "1", + "ColumnNullable": "true" + } + ], + "TableColumnsTotal": "8" + } + ] +} diff --git a/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_settings.json b/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_settings.json index 23a0ff33ca8..79334860e40 100644 --- a/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_settings.json +++ b/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_settings.json @@ -2,7 +2,7 @@ "TargetMetadata": { "TargetSchema": "DELIUS_AUDIT_DMS_POOL", "SupportLobs": false, - "BatchApplyEnabled": true, + "BatchApplyEnabled": true, "TaskRecoveryTableEnabled": false }, "FullLoadSettings": { diff --git a/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_table_mapping.json b/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_table_mapping.json index 82327da86a1..7f713e22f21 100644 --- a/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_table_mapping.json +++ b/terraform/environments/delius-core/files/audited_interaction_checksum_inbound_table_mapping.json @@ -1,33 +1,33 @@ { - "rules": [ - { - "rule-type": "selection", - "rule-id": "50", - "rule-name": "select_clientdb_audit_data_checksum", - "object-locator": { - "schema-name": "DELIUS_AUDIT_DMS_POOL", - "table-name": "AUDITED_INTERACTION_CHECKSUM" - }, - "rule-action": "include" - }, - { - "rule-type": "transformation", - "rule-id": "51", - "rule-name": "set_not_validated", - "rule-target": "column", - "object-locator": { - "schema-name": "DELIUS_AUDIT_DMS_POOL", - "table-name": "AUDITED_INTERACTION_CHECKSUM" - }, - "rule-action": "add-column", - "value": "CHECKSUM_VALIDATED", - "expression": "'N'", - "old-value": null, - "data-type": { - "type": "string", - "length": "1", - "scale": "" - } - } - ] -} \ No newline at end of file + "rules": [ + { + "rule-type": "selection", + "rule-id": "50", + "rule-name": "select_clientdb_audit_data_checksum", + "object-locator": { + "schema-name": "DELIUS_AUDIT_DMS_POOL", + "table-name": "AUDITED_INTERACTION_CHECKSUM" + }, + "rule-action": "include" + }, + { + "rule-type": "transformation", + "rule-id": "51", + "rule-name": "set_not_validated", + "rule-target": "column", + "object-locator": { + "schema-name": "DELIUS_AUDIT_DMS_POOL", + "table-name": "AUDITED_INTERACTION_CHECKSUM" + }, + "rule-action": "add-column", + "value": "CHECKSUM_VALIDATED", + "expression": "'N'", + "old-value": null, + "data-type": { + "type": "string", + "length": "1", + "scale": "" + } + } + ] +} diff --git a/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_settings.json b/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_settings.json index 23a0ff33ca8..79334860e40 100644 --- a/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_settings.json +++ b/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_settings.json @@ -2,7 +2,7 @@ "TargetMetadata": { "TargetSchema": "DELIUS_AUDIT_DMS_POOL", "SupportLobs": false, - "BatchApplyEnabled": true, + "BatchApplyEnabled": true, "TaskRecoveryTableEnabled": false }, "FullLoadSettings": { diff --git a/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_table_mapping.json b/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_table_mapping.json index ad848d4d1af..873707e2a76 100644 --- a/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_table_mapping.json +++ b/terraform/environments/delius-core/files/audited_interaction_checksum_outbound_table_mapping.json @@ -1,14 +1,14 @@ { - "rules": [ - { - "rule-type": "selection", - "rule-id": "50", - "rule-name": "select_clientdb_audit_data_checksum", - "object-locator": { - "schema-name": "DELIUS_AUDIT_DMS_POOL", - "table-name": "AUDITED_INTERACTION_CHECKSUM" - }, - "rule-action": "include" - } - ] -} \ No newline at end of file + "rules": [ + { + "rule-type": "selection", + "rule-id": "50", + "rule-name": "select_clientdb_audit_data_checksum", + "object-locator": { + "schema-name": "DELIUS_AUDIT_DMS_POOL", + "table-name": "AUDITED_INTERACTION_CHECKSUM" + }, + "rule-action": "include" + } + ] +} diff --git a/terraform/environments/delius-core/files/audited_interaction_inbound_settings.json b/terraform/environments/delius-core/files/audited_interaction_inbound_settings.json index 20f65c1ac0c..cbfecba270f 100644 --- a/terraform/environments/delius-core/files/audited_interaction_inbound_settings.json +++ b/terraform/environments/delius-core/files/audited_interaction_inbound_settings.json @@ -1,37 +1,36 @@ { - "TargetMetadata": { - "TargetSchema": "DELIUS_AUDIT_DMS_POOL", - "SupportLobs": false, - "BatchApplyEnabled": true - }, - "FullLoadSettings": { - "TargetTablePrepMode": "DO_NOTHING", - "CreatePkAfterFullLoad": false, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CommitRate": 10000 - }, - "ControlTablesSettings": { - "ControlSchema": "DELIUS_AUDIT_DMS_POOL" - }, - "ErrorBehavior": { - "RecoverableErrorThrottlingMax": 1800, - "RecoverableErrorStopRetryAfterThrottlingMax": false - }, - "Logging": { - "EnableLogging": true, - "LogComponents": [ - { - "Id": "SOURCE_UNLOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_LOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - } - ] - } + "TargetMetadata": { + "TargetSchema": "DELIUS_AUDIT_DMS_POOL", + "SupportLobs": false, + "BatchApplyEnabled": true + }, + "FullLoadSettings": { + "TargetTablePrepMode": "DO_NOTHING", + "CreatePkAfterFullLoad": false, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CommitRate": 10000 + }, + "ControlTablesSettings": { + "ControlSchema": "DELIUS_AUDIT_DMS_POOL" + }, + "ErrorBehavior": { + "RecoverableErrorThrottlingMax": 1800, + "RecoverableErrorStopRetryAfterThrottlingMax": false + }, + "Logging": { + "EnableLogging": true, + "LogComponents": [ + { + "Id": "SOURCE_UNLOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_LOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + } + ] } - \ No newline at end of file +} diff --git a/terraform/environments/delius-core/files/audited_interaction_inbound_table_mapping.json b/terraform/environments/delius-core/files/audited_interaction_inbound_table_mapping.json index 548bf5bc0f7..4837514b7ae 100644 --- a/terraform/environments/delius-core/files/audited_interaction_inbound_table_mapping.json +++ b/terraform/environments/delius-core/files/audited_interaction_inbound_table_mapping.json @@ -1,39 +1,39 @@ { - "rules": [ - { - "rule-type": "selection", - "rule-id": "401", - "rule-name": "select_locally_populated_audit", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "AUDITED_INTERACTION" - }, - "rule-action": "include" - }, - { - "rule-type": "transformation", - "rule-id": "402", - "rule-name": "rename_to_staging_schema", - "rule-target": "schema", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA" - }, - "rule-action": "rename", - "value": "DELIUS_AUDIT_DMS_POOL", - "old-value": null - }, - { - "rule-type": "transformation", - "rule-id": "403", - "rule-name": "rename_to_staging_table", - "rule-target": "table", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "AUDITED_INTERACTION" - }, - "rule-action": "rename", - "value": "DMS_AUDITED_INTERACTION", - "old-value": null - } - ] -} \ No newline at end of file + "rules": [ + { + "rule-type": "selection", + "rule-id": "401", + "rule-name": "select_locally_populated_audit", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "AUDITED_INTERACTION" + }, + "rule-action": "include" + }, + { + "rule-type": "transformation", + "rule-id": "402", + "rule-name": "rename_to_staging_schema", + "rule-target": "schema", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA" + }, + "rule-action": "rename", + "value": "DELIUS_AUDIT_DMS_POOL", + "old-value": null + }, + { + "rule-type": "transformation", + "rule-id": "403", + "rule-name": "rename_to_staging_table", + "rule-target": "table", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "AUDITED_INTERACTION" + }, + "rule-action": "rename", + "value": "DMS_AUDITED_INTERACTION", + "old-value": null + } + ] +} diff --git a/terraform/environments/delius-core/files/audited_interaction_outbound_settings.json b/terraform/environments/delius-core/files/audited_interaction_outbound_settings.json index 20f65c1ac0c..cbfecba270f 100644 --- a/terraform/environments/delius-core/files/audited_interaction_outbound_settings.json +++ b/terraform/environments/delius-core/files/audited_interaction_outbound_settings.json @@ -1,37 +1,36 @@ { - "TargetMetadata": { - "TargetSchema": "DELIUS_AUDIT_DMS_POOL", - "SupportLobs": false, - "BatchApplyEnabled": true - }, - "FullLoadSettings": { - "TargetTablePrepMode": "DO_NOTHING", - "CreatePkAfterFullLoad": false, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CommitRate": 10000 - }, - "ControlTablesSettings": { - "ControlSchema": "DELIUS_AUDIT_DMS_POOL" - }, - "ErrorBehavior": { - "RecoverableErrorThrottlingMax": 1800, - "RecoverableErrorStopRetryAfterThrottlingMax": false - }, - "Logging": { - "EnableLogging": true, - "LogComponents": [ - { - "Id": "SOURCE_UNLOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_LOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - } - ] - } + "TargetMetadata": { + "TargetSchema": "DELIUS_AUDIT_DMS_POOL", + "SupportLobs": false, + "BatchApplyEnabled": true + }, + "FullLoadSettings": { + "TargetTablePrepMode": "DO_NOTHING", + "CreatePkAfterFullLoad": false, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CommitRate": 10000 + }, + "ControlTablesSettings": { + "ControlSchema": "DELIUS_AUDIT_DMS_POOL" + }, + "ErrorBehavior": { + "RecoverableErrorThrottlingMax": 1800, + "RecoverableErrorStopRetryAfterThrottlingMax": false + }, + "Logging": { + "EnableLogging": true, + "LogComponents": [ + { + "Id": "SOURCE_UNLOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_LOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + } + ] } - \ No newline at end of file +} diff --git a/terraform/environments/delius-core/files/business_interaction_inbound_settings.json b/terraform/environments/delius-core/files/business_interaction_inbound_settings.json index 37e80505c54..d4dd45690ae 100644 --- a/terraform/environments/delius-core/files/business_interaction_inbound_settings.json +++ b/terraform/environments/delius-core/files/business_interaction_inbound_settings.json @@ -2,7 +2,7 @@ "TargetMetadata": { "TargetSchema": "DELIUS_AUDIT_DMS_POOL", "SupportLobs": false, - "BatchApplyEnabled": true, + "BatchApplyEnabled": true, "TaskRecoveryTableEnabled": false }, "FullLoadSettings": { diff --git a/terraform/environments/delius-core/files/business_interaction_inbound_table_mapping.json b/terraform/environments/delius-core/files/business_interaction_inbound_table_mapping.json index 3a51e911607..6094d493fe5 100644 --- a/terraform/environments/delius-core/files/business_interaction_inbound_table_mapping.json +++ b/terraform/environments/delius-core/files/business_interaction_inbound_table_mapping.json @@ -1,59 +1,56 @@ { - "rules": [ - { - "rule-type": "selection", - "rule-id": "10", - "rule-name": "select_business_interactions", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "BUSINESS_INTERACTION" - }, - "rule-action": "include" - }, - { - "rule-type": "transformation", - "rule-id": "12", - "rule-name": "rename_to_staging_schema", - "rule-target": "schema", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA" - }, - "rule-action": "rename", - "value": "DELIUS_AUDIT_DMS_POOL", - "old-value": null - }, - { - "rule-type": "transformation", - "rule-id": "13", - "rule-name": "rename_to_staging_table", - "rule-target": "table", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "BUSINESS_INTERACTION" - }, - "rule-action": "rename", - "value": "STAGE_BUSINESS_INTERACTION", - "old-value": null - }, - { - "rule-type": "transformation", - "rule-id": "14", - "rule-name": "stage_business_interaction_pk", - "rule-target": "table", - "object-locator": { - "schema-name": "DELIUS_AUDIT_DMS_POOL", - "table-name": "STAGE_BUSINESS_INTERACTION" - }, - "rule-action": "define-primary-key", - "value": null, - "old-value": null, - "primary-key-def": { - "name": "PK_STAGE_BUSINESS_INTERACTION", - "columns": [ - "CLIENT_DB", - "BUSINESS_INTERACTION_ID" - ] - } - } - ] -} \ No newline at end of file + "rules": [ + { + "rule-type": "selection", + "rule-id": "10", + "rule-name": "select_business_interactions", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "BUSINESS_INTERACTION" + }, + "rule-action": "include" + }, + { + "rule-type": "transformation", + "rule-id": "12", + "rule-name": "rename_to_staging_schema", + "rule-target": "schema", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA" + }, + "rule-action": "rename", + "value": "DELIUS_AUDIT_DMS_POOL", + "old-value": null + }, + { + "rule-type": "transformation", + "rule-id": "13", + "rule-name": "rename_to_staging_table", + "rule-target": "table", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "BUSINESS_INTERACTION" + }, + "rule-action": "rename", + "value": "STAGE_BUSINESS_INTERACTION", + "old-value": null + }, + { + "rule-type": "transformation", + "rule-id": "14", + "rule-name": "stage_business_interaction_pk", + "rule-target": "table", + "object-locator": { + "schema-name": "DELIUS_AUDIT_DMS_POOL", + "table-name": "STAGE_BUSINESS_INTERACTION" + }, + "rule-action": "define-primary-key", + "value": null, + "old-value": null, + "primary-key-def": { + "name": "PK_STAGE_BUSINESS_INTERACTION", + "columns": ["CLIENT_DB", "BUSINESS_INTERACTION_ID"] + } + } + ] +} diff --git a/terraform/environments/delius-core/files/business_interaction_outbound_settings.json b/terraform/environments/delius-core/files/business_interaction_outbound_settings.json index 37e80505c54..d4dd45690ae 100644 --- a/terraform/environments/delius-core/files/business_interaction_outbound_settings.json +++ b/terraform/environments/delius-core/files/business_interaction_outbound_settings.json @@ -2,7 +2,7 @@ "TargetMetadata": { "TargetSchema": "DELIUS_AUDIT_DMS_POOL", "SupportLobs": false, - "BatchApplyEnabled": true, + "BatchApplyEnabled": true, "TaskRecoveryTableEnabled": false }, "FullLoadSettings": { diff --git a/terraform/environments/delius-core/files/user_external_table_definition.json b/terraform/environments/delius-core/files/user_external_table_definition.json index eed0a55fed2..d1966941bd1 100644 --- a/terraform/environments/delius-core/files/user_external_table_definition.json +++ b/terraform/environments/delius-core/files/user_external_table_definition.json @@ -1,196 +1,195 @@ { - "TableCount": 2, - "Tables": [ - { - "TableName": "USER_", - "TablePath": "DELIUS_APP_SCHEMA/USER_/", - "TableOwner": "DELIUS_APP_SCHEMA", - "TableColumns": [ - { - "ColumnName": "DMS_CDC_TIMESTAMP", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", - "ColumnNullable": "false", - "ColumnScale": "6" - }, - { - "ColumnName": "USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "STAFF_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "PROVIDER_EMPLOYEE_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "SURNAME", - "ColumnType": "STRING", - "ColumnLength": "35", - "ColumnNullable": "false" - }, - { - "ColumnName": "FORENAME", - "ColumnType": "STRING", - "ColumnLength": "35", - "ColumnNullable": "false" - }, - { - "ColumnName": "FORENAME2", - "ColumnType": "STRING", - "ColumnLength": "35", - "ColumnNullable": "true" - }, - { - "ColumnName": "END_DATE", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "true" - }, - { - "ColumnName": "NOTES", - "ColumnType": "CLOB", - "ColumnLength": "32000", - "ColumnNullable": "true" - }, - { - "ColumnName": "ROW_VERSION", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "DISTINGUISHED_NAME", - "ColumnType": "STRING", - "ColumnLength": "500", - "ColumnNullable": "false" - }, - { - "ColumnName": "EXTERNAL_PROVIDER_EMPLOYEEFLAG", - "ColumnType": "STRING", - "ColumnLength": "1", - "ColumnNullable": "true" - }, - { - "ColumnName": "EXTERNAL_PROVIDER_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "TRAINING_SESSION_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "PRIVATE", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "ORGANISATION_ID", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "SC_PROVIDER_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "CREATED_BY_USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "CREATED_DATETIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "true" - }, - { - "ColumnName": "LAST_UPDATED_USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - }, - { - "ColumnName": "LAST_UPDATED_DATETIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "true" - }, - { - "ColumnName": "LAST_ACCESSED_DATETIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "true" - } - ], - "TableColumnsTotal": "22" - }, - { - "TableName": "PROBATION_AREA_USER", - "TablePath": "DELIUS_APP_SCHEMA/PROBATION_AREA_USER/", - "TableOwner": "DELIUS_APP_SCHEMA", - "TableColumns": [ - { - "ColumnName": "DMS_CDC_TIMESTAMP", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", - "ColumnNullable": "false", - "ColumnScale": "6" - }, - { - "ColumnName": "USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "PROBATION_AREA_ID", - "ColumnType": "INT8", - "ColumnNullable": "false", - "ColumnIsPk": "true" - }, - { - "ColumnName": "ROW_VERSION", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "CREATED_DATETIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "false" - }, - { - "ColumnName": "CREATED_BY_USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "LAST_UPDATED_DATETIME", - "ColumnType": "DATETIME", - "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", - "ColumnNullable": "false" - }, - { - "ColumnName": "LAST_UPDATED_USER_ID", - "ColumnType": "INT8", - "ColumnNullable": "false" - }, - { - "ColumnName": "TRAINING_SESSION_ID", - "ColumnType": "INT8", - "ColumnNullable": "true" - } - ], - "TableColumnsTotal": "9" - } - ] - } - \ No newline at end of file + "TableCount": 2, + "Tables": [ + { + "TableName": "USER_", + "TablePath": "DELIUS_APP_SCHEMA/USER_/", + "TableOwner": "DELIUS_APP_SCHEMA", + "TableColumns": [ + { + "ColumnName": "DMS_CDC_TIMESTAMP", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", + "ColumnNullable": "false", + "ColumnScale": "6" + }, + { + "ColumnName": "USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "STAFF_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "PROVIDER_EMPLOYEE_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "SURNAME", + "ColumnType": "STRING", + "ColumnLength": "35", + "ColumnNullable": "false" + }, + { + "ColumnName": "FORENAME", + "ColumnType": "STRING", + "ColumnLength": "35", + "ColumnNullable": "false" + }, + { + "ColumnName": "FORENAME2", + "ColumnType": "STRING", + "ColumnLength": "35", + "ColumnNullable": "true" + }, + { + "ColumnName": "END_DATE", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "true" + }, + { + "ColumnName": "NOTES", + "ColumnType": "CLOB", + "ColumnLength": "32000", + "ColumnNullable": "true" + }, + { + "ColumnName": "ROW_VERSION", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "DISTINGUISHED_NAME", + "ColumnType": "STRING", + "ColumnLength": "500", + "ColumnNullable": "false" + }, + { + "ColumnName": "EXTERNAL_PROVIDER_EMPLOYEEFLAG", + "ColumnType": "STRING", + "ColumnLength": "1", + "ColumnNullable": "true" + }, + { + "ColumnName": "EXTERNAL_PROVIDER_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "TRAINING_SESSION_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "PRIVATE", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "ORGANISATION_ID", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "SC_PROVIDER_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "CREATED_BY_USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "CREATED_DATETIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "true" + }, + { + "ColumnName": "LAST_UPDATED_USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + }, + { + "ColumnName": "LAST_UPDATED_DATETIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "true" + }, + { + "ColumnName": "LAST_ACCESSED_DATETIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "true" + } + ], + "TableColumnsTotal": "22" + }, + { + "TableName": "PROBATION_AREA_USER", + "TablePath": "DELIUS_APP_SCHEMA/PROBATION_AREA_USER/", + "TableOwner": "DELIUS_APP_SCHEMA", + "TableColumns": [ + { + "ColumnName": "DMS_CDC_TIMESTAMP", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss.F", + "ColumnNullable": "false", + "ColumnScale": "6" + }, + { + "ColumnName": "USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "PROBATION_AREA_ID", + "ColumnType": "INT8", + "ColumnNullable": "false", + "ColumnIsPk": "true" + }, + { + "ColumnName": "ROW_VERSION", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "CREATED_DATETIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "false" + }, + { + "ColumnName": "CREATED_BY_USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "LAST_UPDATED_DATETIME", + "ColumnType": "DATETIME", + "ColumnDateFormat": "YYYY-MM-dd HH:mm:ss", + "ColumnNullable": "false" + }, + { + "ColumnName": "LAST_UPDATED_USER_ID", + "ColumnType": "INT8", + "ColumnNullable": "false" + }, + { + "ColumnName": "TRAINING_SESSION_ID", + "ColumnType": "INT8", + "ColumnNullable": "true" + } + ], + "TableColumnsTotal": "9" + } + ] +} diff --git a/terraform/environments/delius-core/files/user_inbound_settings.json b/terraform/environments/delius-core/files/user_inbound_settings.json index 3d907c9cf3f..31d1b920e59 100644 --- a/terraform/environments/delius-core/files/user_inbound_settings.json +++ b/terraform/environments/delius-core/files/user_inbound_settings.json @@ -1,56 +1,56 @@ { - "TargetMetadata": { - "ParallelApplyBufferSize": 0, - "ParallelApplyQueuesPerThread": 0, - "ParallelApplyThreads": 0, - "TargetSchema": "DELIUS_APP_SCHEMA", - "InlineLobMaxSize": 0, - "ParallelLoadQueuesPerThread": 0, - "SupportLobs": true, - "LobChunkSize": 64, - "TaskRecoveryTableEnabled": false, - "ParallelLoadThreads": 0, - "LobMaxSize": 0, - "BatchApplyEnabled": false, - "FullLobMode": true, - "LimitedSizeLobMode": false, - "LoadMaxFileSize": 0, - "ParallelLoadBufferSize": 0 - }, - "FullLoadSettings": { - "TargetTablePrepMode": "DO_NOTHING", - "CreatePkAfterFullLoad": false, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CommitRate": 10000 - }, - "ControlTablesSettings": { - "historyTimeslotInMinutes": 5, - "HistoryTimeslotInMinutes": 5, - "StatusTableEnabled": false, - "SuspendedTablesTableEnabled": true, - "HistoryTableEnabled": false, - "ControlSchema": "DELIUS_AUDIT_DMS_POOL", - "FullLoadExceptionTableEnabled": false - }, - "ErrorBehavior": { - "RecoverableErrorThrottlingMax": 1800, - "RecoverableErrorStopRetryAfterThrottlingMax": false, - "ApplyErrorInsertPolicy": "INSERT_RECORD" - }, - "Logging": { - "EnableLogging": true, - "LogComponents": [ - { - "Id": "SOURCE_UNLOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_LOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - } - ] - } + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "DELIUS_APP_SCHEMA", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 0, + "BatchApplyEnabled": false, + "FullLobMode": true, + "LimitedSizeLobMode": false, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "FullLoadSettings": { + "TargetTablePrepMode": "DO_NOTHING", + "CreatePkAfterFullLoad": false, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CommitRate": 10000 + }, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": true, + "HistoryTableEnabled": false, + "ControlSchema": "DELIUS_AUDIT_DMS_POOL", + "FullLoadExceptionTableEnabled": false + }, + "ErrorBehavior": { + "RecoverableErrorThrottlingMax": 1800, + "RecoverableErrorStopRetryAfterThrottlingMax": false, + "ApplyErrorInsertPolicy": "INSERT_RECORD" + }, + "Logging": { + "EnableLogging": true, + "LogComponents": [ + { + "Id": "SOURCE_UNLOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_LOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + } + ] } +} diff --git a/terraform/environments/delius-core/files/user_inbound_table_mapping.json b/terraform/environments/delius-core/files/user_inbound_table_mapping.json index 09810566427..f8a2e966334 100644 --- a/terraform/environments/delius-core/files/user_inbound_table_mapping.json +++ b/terraform/environments/delius-core/files/user_inbound_table_mapping.json @@ -1,26 +1,26 @@ { - "rules": [ - { - "rule-type": "selection", - "rule-id": "30", - "rule-name": "select_probation_area_user", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "PROBATION_AREA_USER" - }, - "rule-action": "include", - "filters": [] - }, - { - "rule-type": "selection", - "rule-id": "31", - "rule-name": "select_user_", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "USER_" - }, - "rule-action": "include", - "filters": [] - } - ] -} \ No newline at end of file + "rules": [ + { + "rule-type": "selection", + "rule-id": "30", + "rule-name": "select_probation_area_user", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "PROBATION_AREA_USER" + }, + "rule-action": "include", + "filters": [] + }, + { + "rule-type": "selection", + "rule-id": "31", + "rule-name": "select_user_", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "USER_" + }, + "rule-action": "include", + "filters": [] + } + ] +} diff --git a/terraform/environments/delius-core/files/user_outbound_settings.json b/terraform/environments/delius-core/files/user_outbound_settings.json index 3d907c9cf3f..31d1b920e59 100644 --- a/terraform/environments/delius-core/files/user_outbound_settings.json +++ b/terraform/environments/delius-core/files/user_outbound_settings.json @@ -1,56 +1,56 @@ { - "TargetMetadata": { - "ParallelApplyBufferSize": 0, - "ParallelApplyQueuesPerThread": 0, - "ParallelApplyThreads": 0, - "TargetSchema": "DELIUS_APP_SCHEMA", - "InlineLobMaxSize": 0, - "ParallelLoadQueuesPerThread": 0, - "SupportLobs": true, - "LobChunkSize": 64, - "TaskRecoveryTableEnabled": false, - "ParallelLoadThreads": 0, - "LobMaxSize": 0, - "BatchApplyEnabled": false, - "FullLobMode": true, - "LimitedSizeLobMode": false, - "LoadMaxFileSize": 0, - "ParallelLoadBufferSize": 0 - }, - "FullLoadSettings": { - "TargetTablePrepMode": "DO_NOTHING", - "CreatePkAfterFullLoad": false, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CommitRate": 10000 - }, - "ControlTablesSettings": { - "historyTimeslotInMinutes": 5, - "HistoryTimeslotInMinutes": 5, - "StatusTableEnabled": false, - "SuspendedTablesTableEnabled": true, - "HistoryTableEnabled": false, - "ControlSchema": "DELIUS_AUDIT_DMS_POOL", - "FullLoadExceptionTableEnabled": false - }, - "ErrorBehavior": { - "RecoverableErrorThrottlingMax": 1800, - "RecoverableErrorStopRetryAfterThrottlingMax": false, - "ApplyErrorInsertPolicy": "INSERT_RECORD" - }, - "Logging": { - "EnableLogging": true, - "LogComponents": [ - { - "Id": "SOURCE_UNLOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_LOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - } - ] - } + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "DELIUS_APP_SCHEMA", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 0, + "BatchApplyEnabled": false, + "FullLobMode": true, + "LimitedSizeLobMode": false, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "FullLoadSettings": { + "TargetTablePrepMode": "DO_NOTHING", + "CreatePkAfterFullLoad": false, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CommitRate": 10000 + }, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": true, + "HistoryTableEnabled": false, + "ControlSchema": "DELIUS_AUDIT_DMS_POOL", + "FullLoadExceptionTableEnabled": false + }, + "ErrorBehavior": { + "RecoverableErrorThrottlingMax": 1800, + "RecoverableErrorStopRetryAfterThrottlingMax": false, + "ApplyErrorInsertPolicy": "INSERT_RECORD" + }, + "Logging": { + "EnableLogging": true, + "LogComponents": [ + { + "Id": "SOURCE_UNLOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_LOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + } + ] } +} diff --git a/terraform/environments/delius-core/files/user_outbound_table_mapping.json b/terraform/environments/delius-core/files/user_outbound_table_mapping.json index 09810566427..f8a2e966334 100644 --- a/terraform/environments/delius-core/files/user_outbound_table_mapping.json +++ b/terraform/environments/delius-core/files/user_outbound_table_mapping.json @@ -1,26 +1,26 @@ { - "rules": [ - { - "rule-type": "selection", - "rule-id": "30", - "rule-name": "select_probation_area_user", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "PROBATION_AREA_USER" - }, - "rule-action": "include", - "filters": [] - }, - { - "rule-type": "selection", - "rule-id": "31", - "rule-name": "select_user_", - "object-locator": { - "schema-name": "DELIUS_APP_SCHEMA", - "table-name": "USER_" - }, - "rule-action": "include", - "filters": [] - } - ] -} \ No newline at end of file + "rules": [ + { + "rule-type": "selection", + "rule-id": "30", + "rule-name": "select_probation_area_user", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "PROBATION_AREA_USER" + }, + "rule-action": "include", + "filters": [] + }, + { + "rule-type": "selection", + "rule-id": "31", + "rule-name": "select_user_", + "object-locator": { + "schema-name": "DELIUS_APP_SCHEMA", + "table-name": "USER_" + }, + "rule-action": "include", + "filters": [] + } + ] +} diff --git a/terraform/environments/delius-core/locals.tf b/terraform/environments/delius-core/locals.tf index 9d89ff38810..2fdec2d9aee 100644 --- a/terraform/environments/delius-core/locals.tf +++ b/terraform/environments/delius-core/locals.tf @@ -23,10 +23,10 @@ locals { # Define a mapping of delius_environments to DMS configuration for that environment. We include the ID of the AWS # account which hosts that particular delius_environment. env_name_to_dms_config_map = { - "dev" = merge({dms_config = local.dms_config_dev}, {account_id = try(local.environment_management.account_ids["delius-core-development"],null)}) - "test" = merge({dms_config = local.dms_config_test}, {account_id = try(local.environment_management.account_ids["delius-core-test"],null)}) - "stage" = merge({dms_config = local.dms_config_stage}, {account_id = try(local.environment_management.account_ids["delius-core-preproduction"],null)}) - "preprod" = merge({dms_config = local.dms_config_preprod}, {account_id = try(local.environment_management.account_ids["delius-core-preproduction"],null)}) - } + "dev" = merge({ dms_config = local.dms_config_dev }, { account_id = try(local.environment_management.account_ids["delius-core-development"], null) }) + "test" = merge({ dms_config = local.dms_config_test }, { account_id = try(local.environment_management.account_ids["delius-core-test"], null) }) + "stage" = merge({ dms_config = local.dms_config_stage }, { account_id = try(local.environment_management.account_ids["delius-core-preproduction"], null) }) + "preprod" = merge({ dms_config = local.dms_config_preprod }, { account_id = try(local.environment_management.account_ids["delius-core-preproduction"], null) }) + } } diff --git a/terraform/environments/delius-core/locals_preproduction.tf b/terraform/environments/delius-core/locals_preproduction.tf index f56eeae0209..f3ed49ac35c 100644 --- a/terraform/environments/delius-core/locals_preproduction.tf +++ b/terraform/environments/delius-core/locals_preproduction.tf @@ -98,8 +98,8 @@ locals { } ldap = { - image_tag = "replace_me" - container_port = 389 + image_tag = "replace_me" + container_port = 389 slapd_log_level = "replace_me" } diff --git a/terraform/environments/delius-core/locals_stage.tf b/terraform/environments/delius-core/locals_stage.tf index 51d80ee6787..d7a9c69c8f4 100644 --- a/terraform/environments/delius-core/locals_stage.tf +++ b/terraform/environments/delius-core/locals_stage.tf @@ -97,8 +97,8 @@ locals { } ldap = { - image_tag = "replace_me" - container_port = 389 + image_tag = "replace_me" + container_port = 389 slapd_log_level = "replace_me" } diff --git a/terraform/environments/delius-core/locals_test.tf b/terraform/environments/delius-core/locals_test.tf index 0680d796833..2736cf04011 100644 --- a/terraform/environments/delius-core/locals_test.tf +++ b/terraform/environments/delius-core/locals_test.tf @@ -134,6 +134,6 @@ locals { read_database = "TSTNDA" } user_target_endpoint = {} - is-production = local.is-production + is-production = local.is-production } } diff --git a/terraform/environments/delius-core/modules/components/dms/cloudwatch-alarms.tf b/terraform/environments/delius-core/modules/components/dms/cloudwatch-alarms.tf index 32a5e6a561d..012a39987b5 100644 --- a/terraform/environments/delius-core/modules/components/dms/cloudwatch-alarms.tf +++ b/terraform/environments/delius-core/modules/components/dms/cloudwatch-alarms.tf @@ -5,33 +5,33 @@ resource "aws_sns_topic" "dms_alerting" { } locals { - aws_dms_replication_tasks = merge( - try(var.dms_config.user_target_endpoint.write_database, null) == null ? {} : { - (aws_dms_replication_task.user_inbound_replication[0].replication_task_arn) = aws_dms_replication_task.user_inbound_replication[0].replication_task_id - }, - {for k in keys(local.client_account_map) : - (aws_dms_replication_task.business_interaction_inbound_replication[k].replication_task_arn) => aws_dms_replication_task.business_interaction_inbound_replication[k].replication_task_id - }, - {for k in keys(local.client_account_map) : - (aws_dms_replication_task.audited_interaction_inbound_replication[k].replication_task_arn) => aws_dms_replication_task.audited_interaction_inbound_replication[k].replication_task_id - }, - {for k in keys(local.client_account_map) : - (aws_dms_replication_task.audited_interaction_checksum_inbound_replication[k].replication_task_arn) => aws_dms_replication_task.audited_interaction_checksum_inbound_replication[k].replication_task_id - }, - try(var.dms_config.audit_source_endpoint.read_database, null) == null ? {} : { - (aws_dms_replication_task.audited_interaction_outbound_replication[0].replication_task_arn) = aws_dms_replication_task.audited_interaction_outbound_replication[0].replication_task_id - }, - {for k in keys(local.client_account_map) : - (aws_dms_replication_task.user_outbound_replication[k].replication_task_arn) => aws_dms_replication_task.user_outbound_replication[k].replication_task_id - }, - try(var.dms_config.audit_source_endpoint.read_database, null) == null ? {} : { - (aws_dms_replication_task.business_interaction_outbound_replication[0].replication_task_arn) = aws_dms_replication_task.business_interaction_outbound_replication[0].replication_task_id - }, - try(var.dms_config.audit_source_endpoint.read_database, null) == null ? {} : { - (aws_dms_replication_task.audited_interaction_checksum_outbound_replication[0].replication_task_arn) = aws_dms_replication_task.audited_interaction_checksum_outbound_replication[0].replication_task_id - }, - ) -} + aws_dms_replication_tasks = merge( + try(var.dms_config.user_target_endpoint.write_database, null) == null ? {} : { + (aws_dms_replication_task.user_inbound_replication[0].replication_task_arn) = aws_dms_replication_task.user_inbound_replication[0].replication_task_id + }, + { for k in keys(local.client_account_map) : + (aws_dms_replication_task.business_interaction_inbound_replication[k].replication_task_arn) => aws_dms_replication_task.business_interaction_inbound_replication[k].replication_task_id + }, + { for k in keys(local.client_account_map) : + (aws_dms_replication_task.audited_interaction_inbound_replication[k].replication_task_arn) => aws_dms_replication_task.audited_interaction_inbound_replication[k].replication_task_id + }, + { for k in keys(local.client_account_map) : + (aws_dms_replication_task.audited_interaction_checksum_inbound_replication[k].replication_task_arn) => aws_dms_replication_task.audited_interaction_checksum_inbound_replication[k].replication_task_id + }, + try(var.dms_config.audit_source_endpoint.read_database, null) == null ? {} : { + (aws_dms_replication_task.audited_interaction_outbound_replication[0].replication_task_arn) = aws_dms_replication_task.audited_interaction_outbound_replication[0].replication_task_id + }, + { for k in keys(local.client_account_map) : + (aws_dms_replication_task.user_outbound_replication[k].replication_task_arn) => aws_dms_replication_task.user_outbound_replication[k].replication_task_id + }, + try(var.dms_config.audit_source_endpoint.read_database, null) == null ? {} : { + (aws_dms_replication_task.business_interaction_outbound_replication[0].replication_task_arn) = aws_dms_replication_task.business_interaction_outbound_replication[0].replication_task_id + }, + try(var.dms_config.audit_source_endpoint.read_database, null) == null ? {} : { + (aws_dms_replication_task.audited_interaction_checksum_outbound_replication[0].replication_task_arn) = aws_dms_replication_task.audited_interaction_checksum_outbound_replication[0].replication_task_id + }, + ) +} resource "aws_cloudwatch_metric_alarm" "dms_cdc_latency_source" { for_each = local.aws_dms_replication_tasks diff --git a/terraform/environments/delius-core/modules/components/dms/dms_db_source_endpoints.tf b/terraform/environments/delius-core/modules/components/dms/dms_db_source_endpoints.tf index 7481e68ffb9..531b6ba27ee 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_db_source_endpoints.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_db_source_endpoints.tf @@ -5,45 +5,45 @@ # In client environments the dms_audit_source_endpoint.read_database must be defined # The endpoint for audit (AUDITED_INTERACTION) is the Delius database. resource "aws_dms_endpoint" "dms_audit_source_endpoint_db" { - count = try(var.dms_config.audit_source_endpoint.read_database, null) == null ? 0 : 1 - database_name = var.dms_config.audit_source_endpoint.read_database - endpoint_id = "audit-data-from-${lower(var.dms_config.audit_source_endpoint.read_database)}" - endpoint_type = "source" - engine_name = "oracle" - username = local.dms_audit_username - password = join(",",[jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username],jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) - server_name = join(".",[var.oracle_db_server_names[var.dms_config.audit_source_endpoint.read_host],var.account_config.route53_inner_zone_info.name]) - port = local.oracle_port - extra_connection_attributes = "ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".",[var.oracle_db_server_names[var.dms_config.audit_source_endpoint.read_host],var.account_config.route53_inner_zone_info.name])}:${local.oracle_port}/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" - # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration - ssl_mode = "verify-ca" - certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn - # Ignore subsequent replacement with a valid wallet - lifecycle { + count = try(var.dms_config.audit_source_endpoint.read_database, null) == null ? 0 : 1 + database_name = var.dms_config.audit_source_endpoint.read_database + endpoint_id = "audit-data-from-${lower(var.dms_config.audit_source_endpoint.read_database)}" + endpoint_type = "source" + engine_name = "oracle" + username = local.dms_audit_username + password = join(",", [jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username], jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) + server_name = join(".", [var.oracle_db_server_names[var.dms_config.audit_source_endpoint.read_host], var.account_config.route53_inner_zone_info.name]) + port = local.oracle_port + extra_connection_attributes = "ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".", [var.oracle_db_server_names[var.dms_config.audit_source_endpoint.read_host], var.account_config.route53_inner_zone_info.name])}:${local.oracle_port}/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" + # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration + ssl_mode = "verify-ca" + certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn + # Ignore subsequent replacement with a valid wallet + lifecycle { ignore_changes = [certificate_arn] - } - depends_on = [aws_dms_certificate.empty_oracle_wallet] + } + depends_on = [aws_dms_certificate.empty_oracle_wallet] } # In repository environments the dms_user_source_endpoint.read_database must be defined # The endpoint for user (USER_) is the Delius database. resource "aws_dms_endpoint" "dms_user_source_endpoint_db" { - count = try(var.dms_config.user_source_endpoint.read_database, null) == null ? 0 : 1 - database_name = var.dms_config.user_source_endpoint.read_database - endpoint_id = "user-data-from-${lower(var.dms_config.user_source_endpoint.read_database)}" - endpoint_type = "source" - engine_name = "oracle" - username = local.dms_audit_username - password = join(",",[jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username],jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) - server_name = join(".",[var.oracle_db_server_names[var.dms_config.user_source_endpoint.read_host],var.account_config.route53_inner_zone_info.name]) - port = local.oracle_port - extra_connection_attributes = "ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".",[var.oracle_db_server_names[var.dms_config.user_source_endpoint.read_host],var.account_config.route53_inner_zone_info.name])}:1521/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" - # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration - ssl_mode = "verify-ca" - certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn - # Ignore subsequent replacement with a valid wallet - lifecycle { + count = try(var.dms_config.user_source_endpoint.read_database, null) == null ? 0 : 1 + database_name = var.dms_config.user_source_endpoint.read_database + endpoint_id = "user-data-from-${lower(var.dms_config.user_source_endpoint.read_database)}" + endpoint_type = "source" + engine_name = "oracle" + username = local.dms_audit_username + password = join(",", [jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username], jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) + server_name = join(".", [var.oracle_db_server_names[var.dms_config.user_source_endpoint.read_host], var.account_config.route53_inner_zone_info.name]) + port = local.oracle_port + extra_connection_attributes = "ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".", [var.oracle_db_server_names[var.dms_config.user_source_endpoint.read_host], var.account_config.route53_inner_zone_info.name])}:1521/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" + # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration + ssl_mode = "verify-ca" + certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn + # Ignore subsequent replacement with a valid wallet + lifecycle { ignore_changes = [certificate_arn] - } - depends_on = [aws_dms_certificate.empty_oracle_wallet] + } + depends_on = [aws_dms_certificate.empty_oracle_wallet] } diff --git a/terraform/environments/delius-core/modules/components/dms/dms_db_target_endpoints.tf b/terraform/environments/delius-core/modules/components/dms/dms_db_target_endpoints.tf index 9c3d89e5ceb..00efa0e8749 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_db_target_endpoints.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_db_target_endpoints.tf @@ -1,44 +1,44 @@ # In client environments the dms_user_target_endpoint.write_database must be defined # The endpoint for user (USER_ and PROBATION_AREA_USER) is the Delius primary database. resource "aws_dms_endpoint" "dms_user_target_endpoint_db" { - count = try(var.dms_config.user_target_endpoint.write_database, null) == null ? 0 : 1 - database_name = var.dms_config.user_target_endpoint.write_database - endpoint_id = "user-data-to-${lower(var.dms_config.user_target_endpoint.write_database)}" - endpoint_type = "target" - engine_name = "oracle" - username = local.dms_audit_username - password = join(",",[jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username],jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) - server_name = join(".",[var.oracle_db_server_names["primarydb"],var.account_config.route53_inner_zone_info.name]) - port = local.oracle_port - extra_connection_attributes = "UseDirectPathFullLoad=false;ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".",[var.oracle_db_server_names["primarydb"],var.account_config.route53_inner_zone_info.name])}:1521/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" - # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration - ssl_mode = "verify-ca" - certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn - # Ignore subsequent replacement with a valid wallet - lifecycle { - ignore_changes = [certificate_arn] - } - depends_on = [aws_dms_certificate.empty_oracle_wallet] + count = try(var.dms_config.user_target_endpoint.write_database, null) == null ? 0 : 1 + database_name = var.dms_config.user_target_endpoint.write_database + endpoint_id = "user-data-to-${lower(var.dms_config.user_target_endpoint.write_database)}" + endpoint_type = "target" + engine_name = "oracle" + username = local.dms_audit_username + password = join(",", [jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username], jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) + server_name = join(".", [var.oracle_db_server_names["primarydb"], var.account_config.route53_inner_zone_info.name]) + port = local.oracle_port + extra_connection_attributes = "UseDirectPathFullLoad=false;ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".", [var.oracle_db_server_names["primarydb"], var.account_config.route53_inner_zone_info.name])}:1521/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" + # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration + ssl_mode = "verify-ca" + certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn + # Ignore subsequent replacement with a valid wallet + lifecycle { + ignore_changes = [certificate_arn] + } + depends_on = [aws_dms_certificate.empty_oracle_wallet] } # In repository environments the end point for audit (AUDITED_INTERACTION, BUSINESS_INTERACTION) is the Delius primary database. resource "aws_dms_endpoint" "dms_audit_target_endpoint_db" { - count = try(var.dms_config.audit_target_endpoint.write_database, null) == null ? 0 : 1 - database_name = var.dms_config.audit_target_endpoint.write_database - endpoint_id = "audit-data-to-${lower(var.dms_config.audit_target_endpoint.write_database)}" - endpoint_type = "target" - engine_name = "oracle" - username = local.dms_audit_username - password = join(",",[jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username],jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) - server_name = join(".",[var.oracle_db_server_names["primarydb"],var.account_config.route53_inner_zone_info.name]) - port = local.oracle_port - extra_connection_attributes = "UseDirectPathFullLoad=false;ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".",[var.oracle_db_server_names["primarydb"],var.account_config.route53_inner_zone_info.name])}:1521/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" - # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration - ssl_mode = "verify-ca" - certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn - # Ignore subsequent replacement with a valid wallet - lifecycle { - ignore_changes = [certificate_arn] - } - depends_on = [aws_dms_certificate.empty_oracle_wallet] + count = try(var.dms_config.audit_target_endpoint.write_database, null) == null ? 0 : 1 + database_name = var.dms_config.audit_target_endpoint.write_database + endpoint_id = "audit-data-to-${lower(var.dms_config.audit_target_endpoint.write_database)}" + endpoint_type = "target" + engine_name = "oracle" + username = local.dms_audit_username + password = join(",", [jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username], jsondecode(data.aws_secretsmanager_secret_version.delius_core_application_passwords.secret_string)[local.dms_audit_username]]) + server_name = join(".", [var.oracle_db_server_names["primarydb"], var.account_config.route53_inner_zone_info.name]) + port = local.oracle_port + extra_connection_attributes = "UseDirectPathFullLoad=false;ArchivedLogDestId=1;AdditionalArchivedLogDestId=32;asm_server=${join(".", [var.oracle_db_server_names["primarydb"], var.account_config.route53_inner_zone_info.name])}:1521/+ASM;asm_user=${local.dms_audit_username};UseBFile=true;UseLogminerReader=false;" + # We initially use an empty wallet for encryption - a populated wallet will be added by DMS configuration + ssl_mode = "verify-ca" + certificate_arn = aws_dms_certificate.empty_oracle_wallet.certificate_arn + # Ignore subsequent replacement with a valid wallet + lifecycle { + ignore_changes = [certificate_arn] + } + depends_on = [aws_dms_certificate.empty_oracle_wallet] } \ No newline at end of file diff --git a/terraform/environments/delius-core/modules/components/dms/dms_iam.tf b/terraform/environments/delius-core/modules/components/dms/dms_iam.tf index d1d5b95b54f..a86bc7b7f7d 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_iam.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_iam.tf @@ -48,28 +48,28 @@ resource "aws_iam_role" "dms_s3_writer_role" { } resource "aws_iam_policy" "dms_s3_bucket_writer_policy" { - count = length(keys(local.bucket_map)) > 0 ? 1 : 0 - name = "dms-s3-bucket-writer-policy" - policy = jsonencode({ + count = length(keys(local.bucket_map)) > 0 ? 1 : 0 + name = "dms-s3-bucket-writer-policy" + policy = jsonencode({ Version = "2012-10-17" Statement = [ - { - Effect = "Allow" - Action = [ + { + Effect = "Allow" + Action = [ "s3:PutObject", "s3:PutObjectAcl", "s3:DeleteObject", - "s3:PutObjectTagging" + "s3:PutObjectTagging" ] - Resource = concat([for bucket in values(local.bucket_map) : "arn:aws:s3:::${bucket}/*"],["${module.s3_bucket_dms_destination.bucket.arn}/*"]) - }, - { - Effect = "Allow" - Action = [ + Resource = concat([for bucket in values(local.bucket_map) : "arn:aws:s3:::${bucket}/*"], ["${module.s3_bucket_dms_destination.bucket.arn}/*"]) + }, + { + Effect = "Allow" + Action = [ "s3:ListBucket" ] - Resource = concat([for bucket in values(local.bucket_map) : "arn:aws:s3:::${bucket}"],[module.s3_bucket_dms_destination.bucket.arn]) - } + Resource = concat([for bucket in values(local.bucket_map) : "arn:aws:s3:::${bucket}"], [module.s3_bucket_dms_destination.bucket.arn]) + } ] }) } @@ -101,24 +101,24 @@ resource "aws_iam_role" "dms_s3_reader_role" { # The reader role only provides access to the local bucket, not those in other accounts resource "aws_iam_policy" "dms_s3_bucket_reader_policy" { - name = "dms-s3-bucket-reader-policy" - policy = jsonencode({ + name = "dms-s3-bucket-reader-policy" + policy = jsonencode({ Version = "2012-10-17" Statement = [ - { - Effect = "Allow" - Action = [ + { + Effect = "Allow" + Action = [ "s3:GetObject" ] Resource = ["${module.s3_bucket_dms_destination.bucket.arn}/*"] - }, - { - Effect = "Allow" - Action = [ + }, + { + Effect = "Allow" + Action = [ "s3:ListBucket" ] Resource = [module.s3_bucket_dms_destination.bucket.arn] - } + } ] }) } diff --git a/terraform/environments/delius-core/modules/components/dms/dms_inbound_tasks.tf b/terraform/environments/delius-core/modules/components/dms/dms_inbound_tasks.tf index 850ffebe140..d066f69ec64 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_inbound_tasks.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_inbound_tasks.tf @@ -4,7 +4,7 @@ resource "aws_dms_replication_task" "user_inbound_replication" { count = try(var.dms_config.user_target_endpoint.write_database, null) == null ? 0 : 1 replication_task_id = "${var.env_name}-user-inbound-replication-task-from-${var.dms_config.audit_target_endpoint.write_environment}" - migration_type = "cdc" + migration_type = "cdc" table_mappings = file("files/user_inbound_table_mapping.json") replication_task_settings = file("files/user_inbound_settings.json") @@ -36,11 +36,11 @@ resource "aws_dms_replication_task" "user_inbound_replication" { resource "aws_dms_replication_task" "business_interaction_inbound_replication" { for_each = local.client_account_map replication_task_id = "${var.env_name}-business-interaction-inbound-replication-task-from-${each.key}" - migration_type = "full-load-and-cdc" + migration_type = "full-load-and-cdc" table_mappings = file("files/business_interaction_inbound_table_mapping.json") replication_task_settings = file("files/business_interaction_inbound_settings.json") - + source_endpoint_arn = aws_dms_s3_endpoint.dms_audit_source_endpoint_s3[each.key].endpoint_arn target_endpoint_arn = aws_dms_endpoint.dms_audit_target_endpoint_db[0].endpoint_arn replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn @@ -69,11 +69,11 @@ resource "aws_dms_replication_task" "business_interaction_inbound_replication" { resource "aws_dms_replication_task" "audited_interaction_inbound_replication" { for_each = local.client_account_map replication_task_id = "${var.env_name}-audited-interaction-inbound-replication-task-from-${each.key}" - migration_type = "cdc" + migration_type = "cdc" table_mappings = file("files/audited_interaction_inbound_table_mapping.json") replication_task_settings = file("files/audited_interaction_inbound_settings.json") - + source_endpoint_arn = aws_dms_s3_endpoint.dms_audit_source_endpoint_s3[each.key].endpoint_arn target_endpoint_arn = aws_dms_endpoint.dms_audit_target_endpoint_db[0].endpoint_arn replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn @@ -101,11 +101,11 @@ resource "aws_dms_replication_task" "audited_interaction_inbound_replication" { resource "aws_dms_replication_task" "audited_interaction_checksum_inbound_replication" { for_each = local.client_account_map replication_task_id = "${var.env_name}-audited-interaction-checksum-inbound-replication-task-from-${each.key}" - migration_type = "cdc" + migration_type = "cdc" table_mappings = file("files/audited_interaction_checksum_inbound_table_mapping.json") replication_task_settings = file("files/audited_interaction_checksum_inbound_settings.json") - + source_endpoint_arn = aws_dms_s3_endpoint.dms_audit_source_endpoint_s3[each.key].endpoint_arn target_endpoint_arn = aws_dms_endpoint.dms_audit_target_endpoint_db[0].endpoint_arn replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn diff --git a/terraform/environments/delius-core/modules/components/dms/dms_outbound_tasks.tf b/terraform/environments/delius-core/modules/components/dms/dms_outbound_tasks.tf index 0446e49205a..e89edbeef1b 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_outbound_tasks.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_outbound_tasks.tf @@ -1,25 +1,25 @@ - # We do not fail a replication task but keep retrying every 1800 seconds (RecoverableErrorStopRetryAfterThrottlingMax=false) - # This allows us to resume after downtime on an endpoint but note that this means that errors will not be raised - # and must be monitored independently. +# We do not fail a replication task but keep retrying every 1800 seconds (RecoverableErrorStopRetryAfterThrottlingMax=false) +# This allows us to resume after downtime on an endpoint but note that this means that errors will not be raised +# and must be monitored independently. # Audit outbound replication only happens in client environments resource "aws_dms_replication_task" "audited_interaction_outbound_replication" { count = try(var.dms_config.audit_source_endpoint.read_database, null) == null ? 0 : 1 replication_task_id = "${var.env_name}-audited-interaction-outbound-replication-task-for-${lower(var.dms_config.audit_source_endpoint.read_database)}" - migration_type = "cdc" + migration_type = "cdc" # Even though we have the option to read from a standby database, we always record the name of the *primary* # database against the CLIENT_DB column when writing to the Staging table. This provides consistency # when querying the data as we do not need to know whether the primary or standby was used. - + # We do not fail a replication task but keep retrying every 1800 seconds (RecoverableErrorStopRetryAfterThrottlingMax=false) # This allows us to resume after downtime on an endpoint but note that this means that errors will not be raised # and must be monitored independently. # - table_mappings = templatefile("templates/audited_interaction_outbound_table_mapping.tmpl",{ - client_database = local.audit_source_primary - }) + table_mappings = templatefile("templates/audited_interaction_outbound_table_mapping.tmpl", { + client_database = local.audit_source_primary + }) replication_task_settings = file("files/audited_interaction_outbound_settings.json") @@ -51,7 +51,7 @@ resource "aws_dms_replication_task" "user_outbound_replication" { # records; instead we only CDC user and probation records. If this task is # restarted we should set the restart time accordingly to pick up only changes # to users and probation area records. - migration_type = "cdc" + migration_type = "cdc" table_mappings = file("files/user_outbound_table_mapping.json") replication_task_settings = file("files/user_outbound_settings.json") @@ -84,11 +84,11 @@ resource "aws_dms_replication_task" "business_interaction_outbound_replication" count = try(var.dms_config.audit_source_endpoint.read_database, null) == null ? 0 : 1 replication_task_id = "${var.env_name}-business-interaction-outbound-replication-task-for-${lower(var.dms_config.audit_source_endpoint.read_database)}" # As this is reference data we can simply reload if required (full-load-and-cdc) - migration_type = "full-load-and-cdc" + migration_type = "full-load-and-cdc" - table_mappings = templatefile("templates/business_interaction_outbound_table_mapping.tmpl",{ - client_database = local.audit_source_primary - }) + table_mappings = templatefile("templates/business_interaction_outbound_table_mapping.tmpl", { + client_database = local.audit_source_primary + }) replication_task_settings = file("files/business_interaction_outbound_settings.json") source_endpoint_arn = aws_dms_endpoint.dms_audit_source_endpoint_db[0].endpoint_arn @@ -117,7 +117,7 @@ resource "aws_dms_replication_task" "business_interaction_outbound_replication" resource "aws_dms_replication_task" "audited_interaction_checksum_outbound_replication" { count = try(var.dms_config.audit_source_endpoint.read_database, null) == null ? 0 : 1 replication_task_id = "${var.env_name}-business-interaction-checksum-outbound-replication-task-for-${lower(var.dms_config.audit_source_endpoint.read_database)}" - migration_type = "cdc" + migration_type = "cdc" table_mappings = file("files/audited_interaction_checksum_outbound_table_mapping.json") replication_task_settings = file("files/audited_interaction_checksum_outbound_settings.json") diff --git a/terraform/environments/delius-core/modules/components/dms/dms_s3.tf b/terraform/environments/delius-core/modules/components/dms/dms_s3.tf index 7f6fb65e6b2..4af619106cf 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_s3.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_s3.tf @@ -16,7 +16,7 @@ module "s3_bucket_dms_destination" { source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.1.0" - bucket_name = local.dms_s3_local_bucket_name + bucket_name = local.dms_s3_local_bucket_name versioning_enabled = false providers = { @@ -28,32 +28,32 @@ module "s3_bucket_dms_destination" { # (2) The writer role in all clients feeding to this environment, if this environment is a repository. # (3) The writer role in this environment. bucket_policy_v2 = [{ - effect = "Allow" - principals = { - type = "AWS" - identifiers = flatten(concat( - [for k,v in local.repository_account_map : "arn:aws:iam::${v}:role/${k}-dms-s3-writer-role"], - [for k,v in local.client_account_map : "arn:aws:iam::${v}:role/${k}-dms-s3-writer-role"], - [aws_iam_role.dms_s3_writer_role.arn])) - } - actions = [ - "s3:PutObject", - "s3:PutObjectAcl", - "s3:DeleteObject", - "s3:PutObjectTagging", - "s3:ListBucket" - ] - },{ - effect = "Allow" - principals = { - type = "AWS" - identifiers = [aws_iam_role.dms_s3_reader_role.arn] - } - actions = [ - "s3:GetObject", - "s3:ListBucket" - ] - }] + effect = "Allow" + principals = { + type = "AWS" + identifiers = flatten(concat( + [for k, v in local.repository_account_map : "arn:aws:iam::${v}:role/${k}-dms-s3-writer-role"], + [for k, v in local.client_account_map : "arn:aws:iam::${v}:role/${k}-dms-s3-writer-role"], + [aws_iam_role.dms_s3_writer_role.arn])) + } + actions = [ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:DeleteObject", + "s3:PutObjectTagging", + "s3:ListBucket" + ] + }, { + effect = "Allow" + principals = { + type = "AWS" + identifiers = [aws_iam_role.dms_s3_reader_role.arn] + } + actions = [ + "s3:GetObject", + "s3:ListBucket" + ] + }] lifecycle_rule = [ { diff --git a/terraform/environments/delius-core/modules/components/dms/dms_s3_source_endpoints.tf b/terraform/environments/delius-core/modules/components/dms/dms_s3_source_endpoints.tf index b1510653e6e..17f21d60edf 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_s3_source_endpoints.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_s3_source_endpoints.tf @@ -1,27 +1,27 @@ # The User s3 source endpoint is only required in Client environments resource "aws_dms_s3_endpoint" "dms_user_source_endpoint_s3" { - count = try(var.dms_config.user_target_endpoint.write_database, null) == null ? 0 : 1 - endpoint_id = "s3-staging-of-user-data-from-${var.dms_config.audit_target_endpoint.write_environment}" - endpoint_type = "source" - service_access_role_arn = aws_iam_role.dms_s3_reader_role.arn - bucket_name = module.s3_bucket_dms_destination.bucket.bucket - bucket_folder = "user" - cdc_path = "cdc" - external_table_definition = file("files/user_external_table_definition.json") - timestamp_column_name = "TIMESTAMP" + count = try(var.dms_config.user_target_endpoint.write_database, null) == null ? 0 : 1 + endpoint_id = "s3-staging-of-user-data-from-${var.dms_config.audit_target_endpoint.write_environment}" + endpoint_type = "source" + service_access_role_arn = aws_iam_role.dms_s3_reader_role.arn + bucket_name = module.s3_bucket_dms_destination.bucket.bucket + bucket_folder = "user" + cdc_path = "cdc" + external_table_definition = file("files/user_external_table_definition.json") + timestamp_column_name = "TIMESTAMP" } # The Audit s3 source endpoint is only required in Repository environments. # We name the bucket folder after the write database for the client, since this must always be the name of the client's primary database. # One endpoint is required for each of the clients of that repository. resource "aws_dms_s3_endpoint" "dms_audit_source_endpoint_s3" { - for_each = local.client_account_map - endpoint_id = "s3-staging-of-audit-data-from-${each.key}" - endpoint_type = "source" - service_access_role_arn = aws_iam_role.dms_s3_reader_role.arn - bucket_name = module.s3_bucket_dms_destination.bucket.bucket - bucket_folder = "audit/${var.env_name_to_dms_config_map[each.key].dms_config.user_target_endpoint.write_database}" - cdc_path = "cdc" - external_table_definition = file("files/audit_external_table_definition.json") - timestamp_column_name = "TIMESTAMP" + for_each = local.client_account_map + endpoint_id = "s3-staging-of-audit-data-from-${each.key}" + endpoint_type = "source" + service_access_role_arn = aws_iam_role.dms_s3_reader_role.arn + bucket_name = module.s3_bucket_dms_destination.bucket.bucket + bucket_folder = "audit/${var.env_name_to_dms_config_map[each.key].dms_config.user_target_endpoint.write_database}" + cdc_path = "cdc" + external_table_definition = file("files/audit_external_table_definition.json") + timestamp_column_name = "TIMESTAMP" } \ No newline at end of file diff --git a/terraform/environments/delius-core/modules/components/dms/dms_s3_target_endpoints.tf b/terraform/environments/delius-core/modules/components/dms/dms_s3_target_endpoints.tf index 58b0e63c028..39337ec4a4f 100644 --- a/terraform/environments/delius-core/modules/components/dms/dms_s3_target_endpoints.tf +++ b/terraform/environments/delius-core/modules/components/dms/dms_s3_target_endpoints.tf @@ -5,29 +5,29 @@ # In client environments the dms_audit_source_endpoint.read_database must be defined # The endpoint for audit (AUDITED_INTERACTION) is the Delius database. resource "aws_dms_s3_endpoint" "dms_audit_target_endpoint_s3" { - count = length(local.repository_account_map) - endpoint_id = "s3-staging-of-audit-data-from-${lower(var.dms_config.audit_source_endpoint.read_database)}" - endpoint_type = "target" - service_access_role_arn = aws_iam_role.dms_s3_writer_role.arn - bucket_name = local.bucket_map[var.dms_config.audit_target_endpoint.write_environment] - bucket_folder = "audit/${local.audit_source_primary}" - cdc_path = "cdc" - preserve_transactions = true - timestamp_column_name = "TIMESTAMP" - canned_acl_for_objects = "bucket-owner-full-control" - } + count = length(local.repository_account_map) + endpoint_id = "s3-staging-of-audit-data-from-${lower(var.dms_config.audit_source_endpoint.read_database)}" + endpoint_type = "target" + service_access_role_arn = aws_iam_role.dms_s3_writer_role.arn + bucket_name = local.bucket_map[var.dms_config.audit_target_endpoint.write_environment] + bucket_folder = "audit/${local.audit_source_primary}" + cdc_path = "cdc" + preserve_transactions = true + timestamp_column_name = "TIMESTAMP" + canned_acl_for_objects = "bucket-owner-full-control" +} # In repository environments we must loop through all client environments which write to it, as we # will be pushing user updates to all of these. resource "aws_dms_s3_endpoint" "dms_user_target_endpoint_s3" { - for_each = local.client_account_map - endpoint_id = "s3-staging-of-user-data-from-${lower(var.dms_config.user_source_endpoint.read_database)}-to-${each.key}" - endpoint_type = "target" - service_access_role_arn = aws_iam_role.dms_s3_writer_role.arn - bucket_name = local.bucket_map[each.key] - bucket_folder = "user" - cdc_path = "cdc" - preserve_transactions = true - timestamp_column_name = "TIMESTAMP" - canned_acl_for_objects = "bucket-owner-full-control" - } + for_each = local.client_account_map + endpoint_id = "s3-staging-of-user-data-from-${lower(var.dms_config.user_source_endpoint.read_database)}-to-${each.key}" + endpoint_type = "target" + service_access_role_arn = aws_iam_role.dms_s3_writer_role.arn + bucket_name = local.bucket_map[each.key] + bucket_folder = "user" + cdc_path = "cdc" + preserve_transactions = true + timestamp_column_name = "TIMESTAMP" + canned_acl_for_objects = "bucket-owner-full-control" +} diff --git a/terraform/environments/delius-core/modules/components/dms/locals.tf b/terraform/environments/delius-core/modules/components/dms/locals.tf index 08baec46a2c..6bef7964e7a 100644 --- a/terraform/environments/delius-core/modules/components/dms/locals.tf +++ b/terraform/environments/delius-core/modules/components/dms/locals.tf @@ -1,9 +1,9 @@ locals { - account_id = data.aws_caller_identity.current.account_id - delius_account_id = var.platform_vars.environment_management.account_ids[join("-", ["delius-core", var.account_info.mp_environment])] - oracle_port = "1521" + account_id = data.aws_caller_identity.current.account_id + delius_account_id = var.platform_vars.environment_management.account_ids[join("-", ["delius-core", var.account_info.mp_environment])] + oracle_port = "1521" dms_audit_username = "delius_audit_dms_pool" - + # Although it is recommended to use bucket_prefix rather than bucket_name when creating an S3 bucket # using the modernisation-platform-terraform-s3-bucket repo, this introduces significant complications # in this use case since we need to know the names of the buckets in other accounts, and having @@ -11,28 +11,28 @@ locals { # special case we go against the recommendation and use a fixed name for the bucket in each environment # so that it can be templated and does not need to be looked up. dms_s3_local_bucket_format = "delius-audit-dms-s3-staging-bucket" - dms_s3_local_bucket_name = "${var.env_name}-${local.dms_s3_local_bucket_format}" + dms_s3_local_bucket_name = "${var.env_name}-${local.dms_s3_local_bucket_format}" # If we are reading from a standby database it will have an S1 or S2 suffix - strip this off to get the name of the primary database - audit_source_primary = try(replace(upper(var.dms_config.audit_source_endpoint.read_database),"/S[1-2]$/",""),null) + audit_source_primary = try(replace(upper(var.dms_config.audit_source_endpoint.read_database), "/S[1-2]$/", ""), null) # Create map of repositories used by this environment (where this environment is a client) - repository_account_map = try(var.dms_config.audit_target_endpoint.write_environment, null) == null ? {} : {(var.dms_config.audit_target_endpoint.write_environment) = var.env_name_to_dms_config_map[var.dms_config.audit_target_endpoint.write_environment].account_id} + repository_account_map = try(var.dms_config.audit_target_endpoint.write_environment, null) == null ? {} : { (var.dms_config.audit_target_endpoint.write_environment) = var.env_name_to_dms_config_map[var.dms_config.audit_target_endpoint.write_environment].account_id } # Create map of clients of this environment (where this environment is a repository) - client_account_map = {for delius_environment in keys(var.env_name_to_dms_config_map): - delius_environment => var.env_name_to_dms_config_map[delius_environment].account_id if try(var.env_name_to_dms_config_map[delius_environment].dms_config.audit_target_endpoint.write_environment,null) == var.env_name - } + client_account_map = { for delius_environment in keys(var.env_name_to_dms_config_map) : + delius_environment => var.env_name_to_dms_config_map[delius_environment].account_id if try(var.env_name_to_dms_config_map[delius_environment].dms_config.audit_target_endpoint.write_environment, null) == var.env_name + } client_account_ids = values(local.client_account_map) - + # The bucket_list_target_map is, for this environment, either the repository account or all client accounts. # These will be mutually exclusive since a repository may not be a client. It provides a map # of all possible accounts for which we need to retrieve the S3 bucket names for DMS. - bucket_list_target_map = merge(local.repository_account_map,local.client_account_map) + bucket_list_target_map = merge(local.repository_account_map, local.client_account_map) bucket_map = { - for delius_environment in keys(local.bucket_list_target_map): - delius_environment => "${delius_environment}-${local.dms_s3_local_bucket_format}" + for delius_environment in keys(local.bucket_list_target_map) : + delius_environment => "${delius_environment}-${local.dms_s3_local_bucket_format}" } # dms_s3_writer_account_ids = flatten(compact(concat(local.client_account_ids,[local.dms_repository_account_id]))) diff --git a/terraform/environments/delius-core/modules/components/dms/oracle_wallet.tf b/terraform/environments/delius-core/modules/components/dms/oracle_wallet.tf index ffafe03e30e..cbc388d7535 100644 --- a/terraform/environments/delius-core/modules/components/dms/oracle_wallet.tf +++ b/terraform/environments/delius-core/modules/components/dms/oracle_wallet.tf @@ -6,8 +6,8 @@ # We use base64 encoding of the originally binary wallet # (base64 -i cwallet.sso -o empty_wallet_base64.txt) resource "aws_dms_certificate" "empty_oracle_wallet" { - certificate_id = "empty-oracle-wallet" - certificate_wallet = file("files/empty_wallet_base64.txt") + certificate_id = "empty-oracle-wallet" + certificate_wallet = file("files/empty_wallet_base64.txt") lifecycle { ignore_changes = [certificate_wallet] } diff --git a/terraform/environments/delius-core/modules/components/dms/sg.tf b/terraform/environments/delius-core/modules/components/dms/sg.tf index 334b043226c..dcd1a9d6879 100644 --- a/terraform/environments/delius-core/modules/components/dms/sg.tf +++ b/terraform/environments/delius-core/modules/components/dms/sg.tf @@ -56,12 +56,12 @@ resource "aws_vpc_security_group_ingress_rule" "db_dms_conn_in" { } resource "aws_vpc_security_group_egress_rule" "dms_s3_conn_out" { - security_group_id = aws_security_group.dms.id - description = "Allow outgoing communication between DMS and VPC S3 endpoint" - from_port = 443 - to_port = 443 - ip_protocol = "tcp" - prefix_list_id = data.aws_prefix_list.s3.prefix_list_id + security_group_id = aws_security_group.dms.id + description = "Allow outgoing communication between DMS and VPC S3 endpoint" + from_port = 443 + to_port = 443 + ip_protocol = "tcp" + prefix_list_id = data.aws_prefix_list.s3.prefix_list_id tags = merge(var.tags, { Name = "s3-out" } ) diff --git a/terraform/environments/delius-core/modules/delius_environment/dms.tf b/terraform/environments/delius-core/modules/delius_environment/dms.tf index 03845c1240c..88e34f2e943 100644 --- a/terraform/environments/delius-core/modules/delius_environment/dms.tf +++ b/terraform/environments/delius-core/modules/delius_environment/dms.tf @@ -27,5 +27,5 @@ locals { standbydb1 = try(module.oracle_db_standby[0].oracle_db_server_name, "none"), standbydb2 = try(module.oracle_db_standby[1].oracle_db_server_name, "none") } - + } diff --git a/terraform/environments/delius-core/modules/delius_environment/ldap_ecs.tf b/terraform/environments/delius-core/modules/delius_environment/ldap_ecs.tf index 5438e5690ec..1fec3ae9693 100644 --- a/terraform/environments/delius-core/modules/delius_environment/ldap_ecs.tf +++ b/terraform/environments/delius-core/modules/delius_environment/ldap_ecs.tf @@ -104,7 +104,7 @@ module "ldap_ecs" { description = "Allow all outbound traffic to any IPv4 address" } ] - + nlb_ingress_security_group_ids = [ { port = var.ldap_config.port @@ -353,7 +353,7 @@ resource "aws_acm_certificate_validation" "ldap_external" { } resource "aws_cloudwatch_log_group" "ldap_automation" { - name = "/ecs/ldap-automation-${var.env_name}" + name = "/ecs/ldap-automation-${var.env_name}" retention_in_days = 7 tags = var.tags } \ No newline at end of file diff --git a/terraform/environments/delius-core/modules/delius_environment/variables.tf b/terraform/environments/delius-core/modules/delius_environment/variables.tf index eedc38d57af..b911448b46b 100644 --- a/terraform/environments/delius-core/modules/delius_environment/variables.tf +++ b/terraform/environments/delius-core/modules/delius_environment/variables.tf @@ -42,7 +42,7 @@ variable "ldap_config" { efs_backup_schedule = "default_efs_backup_schedule" efs_backup_retention_period = "default_efs_backup_retention_period" port = 389 - tls_port = 636 + tls_port = 636 } } diff --git a/terraform/environments/delius-core/modules/helpers/delius_microservice/variables.tf b/terraform/environments/delius-core/modules/helpers/delius_microservice/variables.tf index 999281ce66f..51c3cf601fa 100644 --- a/terraform/environments/delius-core/modules/helpers/delius_microservice/variables.tf +++ b/terraform/environments/delius-core/modules/helpers/delius_microservice/variables.tf @@ -625,6 +625,6 @@ variable "nlb_egress_security_group_ids" { variable "system_controls" { description = "The system controls for the container" - type = list - default = [] + type = list(any) + default = [] } diff --git a/terraform/environments/digital-prison-reporting/modules/ec2/variables.tf b/terraform/environments/digital-prison-reporting/modules/ec2/variables.tf index 81e5a21c640..c69bd98bc37 100644 --- a/terraform/environments/digital-prison-reporting/modules/ec2/variables.tf +++ b/terraform/environments/digital-prison-reporting/modules/ec2/variables.tf @@ -24,7 +24,7 @@ variable "aws_region" { } variable "ec2_terminate_behavior" { - type = string + type = string } variable "description" { @@ -40,12 +40,12 @@ variable "tags" { } variable "vpc" { - type = string + type = string } variable "ec2_sec_rules" { description = "A Map of map of security group Rules to associate with" - type = map(object({ + type = map(object({ from_port = number to_port = number protocol = string @@ -81,13 +81,13 @@ variable "ec2_sec_rules" { variable "ec2_sec_rules_source_sec_group" { description = "A Map of security group Rules that allows ingress from a specified security group" - type = map(object({ + type = map(object({ from_port = number to_port = number protocol = string source_security_group_id = string })) - default = {} + default = {} } variable "cidr" { diff --git a/terraform/environments/digital-prison-reporting/modules/pipeline_trigger/variables.tf b/terraform/environments/digital-prison-reporting/modules/pipeline_trigger/variables.tf index e86eafa3046..78085271a63 100644 --- a/terraform/environments/digital-prison-reporting/modules/pipeline_trigger/variables.tf +++ b/terraform/environments/digital-prison-reporting/modules/pipeline_trigger/variables.tf @@ -21,8 +21,8 @@ variable "time_window_mode" { } variable "maximum_window_in_minutes" { - type = number - default = null + type = number + default = null description = "(Optional) The maximum time window in minutes" } diff --git a/terraform/environments/digital-prison-reporting/sg.tf b/terraform/environments/digital-prison-reporting/sg.tf index ebd2878282f..a00ec259001 100644 --- a/terraform/environments/digital-prison-reporting/sg.tf +++ b/terraform/environments/digital-prison-reporting/sg.tf @@ -28,7 +28,7 @@ resource "aws_security_group" "lambda_generic" { resource "aws_security_group_rule" "lambda_ingress_generic" { #checkov:skip=CKV_AWS_23: "Ensure every security group and rule has a description" - + count = local.enable_generic_lambda_sg ? 1 : 0 cidr_blocks = [data.aws_vpc.dpr.cidr_block, ] diff --git a/terraform/environments/electronic-monitoring-data/s3.tf b/terraform/environments/electronic-monitoring-data/s3.tf index f51c1ca1860..9dd088bd212 100644 --- a/terraform/environments/electronic-monitoring-data/s3.tf +++ b/terraform/environments/electronic-monitoring-data/s3.tf @@ -4,7 +4,7 @@ locals { "development" = "dev" } environment_shorthand = lookup(local.environment_map, local.environment) - + bucket_prefix = "emds-${local.environment_shorthand}" } @@ -108,7 +108,7 @@ module "s3-metadata-bucket" { "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, }) - log_prefix = "logs/${local.bucket_prefix}-metadata/" + log_prefix = "logs/${local.bucket_prefix}-metadata/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -197,8 +197,8 @@ module "s3-athena-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/${local.bucket_prefix}-athena-query-results/" + }) + log_prefix = "logs/${local.bucket_prefix}-athena-query-results/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -275,7 +275,7 @@ module "s3-unzipped-files-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) + }) log_prefix = "logs/${local.bucket_prefix}-unzipped-files/" log_partition_date_source = "EventTime" @@ -330,8 +330,8 @@ module "s3-dms-premigrate-assess-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/${local.bucket_prefix}-dms-premigrate-assess/" + }) + log_prefix = "logs/${local.bucket_prefix}-dms-premigrate-assess/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -408,8 +408,8 @@ module "s3-json-directory-structure-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/${local.bucket_prefix}-json-directory-structure/" + }) + log_prefix = "logs/${local.bucket_prefix}-json-directory-structure/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -461,7 +461,7 @@ module "s3-json-directory-structure-bucket" { # ------------------------------------------------------------------------ module "s3-data-bucket" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=f759060" + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=f759060" bucket_prefix = "${local.bucket_prefix}-data-" versioning_enabled = true @@ -484,8 +484,8 @@ module "s3-data-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/${local.bucket_prefix}-data/" + }) + log_prefix = "logs/${local.bucket_prefix}-data/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -538,7 +538,7 @@ module "s3-data-bucket" { module "s3-dms-data-validation-bucket" { source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=f759060" - bucket_prefix = "${local.bucket_prefix}-dms-data-validation-" + bucket_prefix = "${local.bucket_prefix}-dms-data-validation-" versioning_enabled = true # to disable ACLs in preference of BucketOwnership controls as per https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/ set: @@ -561,8 +561,8 @@ module "s3-dms-data-validation-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/${local.bucket_prefix}-dms-data-validation/" + }) + log_prefix = "logs/${local.bucket_prefix}-dms-data-validation/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -639,8 +639,8 @@ module "s3-glue-job-script-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/${local.bucket_prefix}-glue-job-store/" + }) + log_prefix = "logs/${local.bucket_prefix}-glue-job-store/" log_partition_date_source = "EventTime" lifecycle_rule = [ @@ -695,7 +695,7 @@ module "s3-glue-job-script-bucket" { module "s3-dms-target-store-bucket" { source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=f759060" - bucket_prefix = "${local.bucket_prefix}-dms-rds-to-parquet-" + bucket_prefix = "${local.bucket_prefix}-dms-rds-to-parquet-" versioning_enabled = true # to disable ACLs in preference of BucketOwnership controls as per https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/ set: @@ -718,8 +718,8 @@ module "s3-dms-target-store-bucket" { "log_bucket_name" : module.s3-logging-bucket.bucket.id, "log_bucket_arn" : module.s3-logging-bucket.bucket.arn, "log_bucket_policy" : module.s3-logging-bucket.bucket_policy.policy, - }) - log_prefix = "logs/dms-target-store/" + }) + log_prefix = "logs/dms-target-store/" log_partition_date_source = "EventTime" lifecycle_rule = [ diff --git a/terraform/environments/wardship/dns_ssl.tf b/terraform/environments/wardship/dns_ssl.tf index 4caa0ea06f6..82af0eb850c 100644 --- a/terraform/environments/wardship/dns_ssl.tf +++ b/terraform/environments/wardship/dns_ssl.tf @@ -16,7 +16,7 @@ resource "aws_acm_certificate" "external" { // Validate Cert based on external route53 fqdn resource "aws_acm_certificate_validation" "external" { certificate_arn = aws_acm_certificate.external.arn - validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn] + validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn] } // Non production zone for validation is network-services (production is application zone) @@ -26,23 +26,23 @@ resource "aws_route53_record" "cert_validation" { for_each = { for dvo in aws_acm_certificate.external.domain_validation_options : dvo.domain_name => { - name = dvo.resource_record_name - type = dvo.resource_record_type - value = dvo.resource_record_value + name = dvo.resource_record_name + type = dvo.resource_record_type + value = dvo.resource_record_value } } allow_overwrite = true - name = each.value.name - records = [each.value.value] - ttl = 300 - type = each.value.type - zone_id = local.is-production ? data.aws_route53_zone.application_zone.zone_id : data.aws_route53_zone.network-services.zone_id + name = each.value.name + records = [each.value.value] + ttl = 300 + type = each.value.type + zone_id = local.is-production ? data.aws_route53_zone.application_zone.zone_id : data.aws_route53_zone.network-services.zone_id } // sub-domain validation only required for non-production sites resource "aws_route53_record" "external_validation_subdomain" { - count = local.is-production ? 0 : 1 + count = local.is-production ? 0 : 1 provider = aws.core-vpc allow_overwrite = true @@ -56,11 +56,11 @@ resource "aws_route53_record" "external_validation_subdomain" { // Route53 DNS record for directing traffic to the service // Provider, zone and name dependent on production or non-production environment resource "aws_route53_record" "external-prod" { - count = local.is-production ? 1 : 0 + count = local.is-production ? 1 : 0 provider = aws.core-network-services - zone_id = data.aws_route53_zone.application_zone.zone_id - name = "wardship-agreements-register.service.justice.gov.uk" - type = "A" + zone_id = data.aws_route53_zone.application_zone.zone_id + name = "wardship-agreements-register.service.justice.gov.uk" + type = "A" alias { name = aws_lb.wardship_lb.dns_name @@ -70,11 +70,11 @@ resource "aws_route53_record" "external-prod" { } resource "aws_route53_record" "external" { - count = local.is-production ? 0 : 1 + count = local.is-production ? 0 : 1 provider = aws.core-vpc - zone_id = data.aws_route53_zone.external.zone_id - name = "${var.networking[0].application}.${var.networking[0].business-unit}-${local.environment}.modernisation-platform.service.justice.gov.uk" - type = "A" + zone_id = data.aws_route53_zone.external.zone_id + name = "${var.networking[0].application}.${var.networking[0].business-unit}-${local.environment}.modernisation-platform.service.justice.gov.uk" + type = "A" alias { name = aws_lb.wardship_lb.dns_name diff --git a/terraform/environments/xhibit-portal/shield.tf b/terraform/environments/xhibit-portal/shield.tf index 64a20a743ad..5389f7c3ada 100644 --- a/terraform/environments/xhibit-portal/shield.tf +++ b/terraform/environments/xhibit-portal/shield.tf @@ -4,7 +4,7 @@ module "shield" { providers = { aws.modernisation-platform = aws.modernisation-platform } - application_name = local.application_name + application_name = local.application_name excluded_protections = ["aec0eb6a-62b1-4433-a854-77fb8b275db5"] resources = { prtg_lb = {