diff --git a/terraform/environments/ccms-ebs/ccms-iam.tf b/terraform/environments/ccms-ebs/ccms-iam.tf index c858ffde922..9f5696f3711 100644 --- a/terraform/environments/ccms-ebs/ccms-iam.tf +++ b/terraform/environments/ccms-ebs/ccms-iam.tf @@ -178,7 +178,6 @@ resource "aws_iam_role_policy_attachment" "rman_to_s3_policy" { policy_arn = aws_iam_policy.rman_to_s3.arn } - # Oracle Licensing policy resource "aws_iam_policy" "oracle_licensing" { name = "oracle_licensing_policy-${local.environment}" @@ -212,3 +211,36 @@ resource "aws_iam_role_policy_attachment" "oracle_licensing_policy" { role = aws_iam_role.role_stsassume_oracle_base.name policy_arn = aws_iam_policy.oracle_licensing.arn } + +# Access to LZ buckets. +resource "aws_iam_policy" "access_to_lz_buckets" { + name = "access_to_lz_buckets-${local.environment}" + description = "Allows licensing metrics to be captured" + + policy = jsonencode( + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Sid" : "AccessToLZBuckets", + "Effect" : "Allow", + "Action" : [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject" + ], + "Resource" : [ + "arn:aws:s3:::laa-ccms-inbound-*", + "arn:aws:s3:::laa-ccms-outbound-*" + ] + } + ] + } + ) +} + +resource "aws_iam_role_policy_attachment" "access_to_lz_buckets_policy" { + role = aws_iam_role.role_stsassume_oracle_base.name + policy_arn = aws_iam_policy.access_to_lz_buckets.arn +} diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json index 668b67640a6..a98e6ff39d1 100644 --- a/terraform/environments/data-platform/application_variables.auto.tfvars.json +++ b/terraform/environments/data-platform/application_variables.auto.tfvars.json @@ -30,10 +30,10 @@ "production": "1.2.1" }, "create_metadata_versions": { - "development": "1.0.6", - "test": "1.0.6", - "preproduction": "1.0.6", - "production": "1.0.6" + "development": "1.0.7", + "test": "1.0.7", + "preproduction": "1.0.7", + "production": "1.0.7" }, "resync_unprocessed_files_versions": { "development": "1.0.2", @@ -46,5 +46,11 @@ "test": "1.0.4", "preproduction": "1.0.4", "production": "1.0.4" + }, + "landing_to_raw_versions": { + "development": "1.0.2", + "test": "1.0.2", + "preproduction": "1.0.2", + "production": "1.0.2" } } diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 67d938bb57e..d3e09416e36 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -1,5 +1,35 @@ # IAM policy documents for lambda functions +data "aws_iam_policy_document" "log_to_bucket" { + statement { + sid = "s3LogAccess" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + resources = [ + "${module.logs_s3_bucket.bucket.arn}", + "${module.logs_s3_bucket.bucket.arn}/*" + ] + } +} + +data "aws_iam_policy_document" "read_metadata" { + statement { + sid = "s3ReadMetadata" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:ListBucket", + ] + resources = [ + "${module.metadata_s3_bucket.bucket.arn}", + "${module.metadata_s3_bucket.bucket.arn}/*" + ] + } +} + data "aws_iam_policy_document" "iam_policy_document_for_docs_lambda" { statement { sid = "LambdaLogGroup" @@ -10,6 +40,8 @@ data "aws_iam_policy_document" "iam_policy_document_for_docs_lambda" { } data "aws_iam_policy_document" "athena_load_lambda_function_policy" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + statement { sid = "AllowLambdaToCreateLogGroup" effect = "Allow" @@ -41,8 +73,10 @@ data "aws_iam_policy_document" "athena_load_lambda_function_policy" { "s3:GetBucketLocation" ] resources = [ - "${module.s3-bucket.bucket.arn}/*", - "${module.s3-bucket.bucket.arn}", + "${module.data_s3_bucket.bucket.arn}/raw/*", + "${module.data_s3_bucket.bucket.arn}/fail/*", + "${module.data_s3_bucket.bucket.arn}/curated/*", + "${module.data_s3_bucket.bucket.arn}", "${module.s3_athena_query_results_bucket.bucket.arn}", "${module.s3_athena_query_results_bucket.bucket.arn}/*" ] @@ -99,27 +133,69 @@ data "aws_iam_policy_document" "athena_load_lambda_function_policy" { } } -data "aws_iam_policy_document" "iam_policy_document_for_authorizer_lambda" { +data "aws_iam_policy_document" "landing_to_raw_lambda_policy" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] + sid = "AllowLambdaToCreateLogGroup" + effect = "Allow" + actions = [ + "logs:CreateLogGroup" + ] + resources = [ + format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) + ] } statement { - sid = "s3LogAccess" + sid = "AllowLambdaToWriteLogsToGroup" effect = "Allow" actions = [ - "s3:GetObject", - "s3:PutObject", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = [ + format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) + ] + } + statement { + sid = "getLandingData" + effect = "Allow" + actions = [ + "s3:GetObject*", + "s3:ListBucket", + ] + resources = [ + "${module.data_landing_s3_bucket.bucket.arn}/*", + "${module.data_landing_s3_bucket.bucket.arn}", + ] + } + statement { + sid = "putRawData" + effect = "Allow" + actions = [ + "s3:PutObject*", + "s3:ListBucket", ] resources = [ - "${module.s3-bucket.bucket.arn}/logs/*" + "${module.data_s3_bucket.bucket.arn}/raw/*", + "${module.data_s3_bucket.bucket.arn}/raw", ] } } +data "aws_iam_policy_document" "iam_policy_document_for_authorizer_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json] + + statement { + sid = "LambdaLogGroup" + effect = "Allow" + actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] + resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] + } +} + data "aws_iam_policy_document" "iam_policy_document_for_get_glue_metadata_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json] statement { sid = "GlueReadOnly" effect = "Allow" @@ -136,40 +212,23 @@ data "aws_iam_policy_document" "iam_policy_document_for_get_glue_metadata_lambda actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] } - statement { - sid = "s3LogAccess" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - ] - resources = [ - "${module.s3-bucket.bucket.arn}/logs/*" - ] - } } data "aws_iam_policy_document" "iam_policy_document_for_presigned_url_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + statement { sid = "GetPutDataObject" effect = "Allow" - actions = ["s3:GetObject", "s3:PutObject"] + actions = ["s3:GetObject", "s3:PutObject", "s3:ListBucket"] resources = [ - "${module.s3-bucket.bucket.arn}/raw_data/*", - "${module.s3-bucket.bucket.arn}/logs/*" + "${module.data_s3_bucket.bucket.arn}/raw/*", + "${module.logs_s3_bucket.bucket.arn}/logs/*", + "${module.data_s3_bucket.bucket.arn}/raw", + "${module.logs_s3_bucket.bucket.arn}/logs", ] } - statement { - sid = "ListExistingDataProducts" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [module.s3-bucket.bucket.arn] - condition { - test = "StringLike" - variable = "s3:prefix" - values = ["metadata/*"] - } - } + statement { sid = "LambdaLogGroup" effect = "Allow" @@ -221,6 +280,7 @@ resource "aws_iam_role_policy_attachment" "attach_allow_invoke_authoriser_lambda # S3 policy +# TO BE REMOVED data "aws_iam_policy_document" "data_platform_product_bucket_policy_document" { statement { sid = "AllowPutFromCiUser" @@ -247,6 +307,157 @@ data "aws_iam_policy_document" "data_platform_product_bucket_policy_document" { type = "AWS" } + condition { + test = "StringNotEquals" + variable = "s3:x-amz-acl" + + values = [ + "bucket-owner-full-control" + ] + } + } +} + +data "aws_iam_policy_document" "data_s3_bucket_policy_document" { + statement { + sid = "AllowPutFromCiUser" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/cicd-member-user"] + } + + actions = ["s3:PutObject", "s3:ListBucket"] + + resources = [module.data_s3_bucket.bucket.arn, "${module.data_s3_bucket.bucket.arn}/*"] + } + + statement { + sid = "DenyNonFullControlObjects" + effect = "Deny" + actions = ["s3:PutObject"] + resources = ["${module.data_s3_bucket.bucket.arn}/*"] + + principals { + identifiers = ["*"] + type = "AWS" + } + + condition { + test = "StringNotEquals" + variable = "s3:x-amz-acl" + + values = [ + "bucket-owner-full-control" + ] + } + } + +} + +data "aws_iam_policy_document" "data_landing_s3_bucket_policy_document" { + statement { + sid = "AllowPutFromCiUser" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/cicd-member-user"] + } + + actions = ["s3:PutObject", "s3:ListBucket"] + + resources = [module.data_landing_s3_bucket.bucket.arn, "${module.data_landing_s3_bucket.bucket.arn}/*"] + } + + statement { + sid = "DenyNonFullControlObjects" + effect = "Deny" + actions = ["s3:PutObject"] + resources = ["${module.data_landing_s3_bucket.bucket.arn}/*"] + + principals { + identifiers = ["*"] + type = "AWS" + } + + condition { + test = "StringNotEquals" + variable = "s3:x-amz-acl" + + values = [ + "bucket-owner-full-control" + ] + } + } + +} + +data "aws_iam_policy_document" "metadata_s3_bucket_policy_document" { + statement { + sid = "AllowPutFromCiUser" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/cicd-member-user"] + } + + actions = ["s3:PutObject", "s3:ListBucket"] + + resources = [module.metadata_s3_bucket.bucket.arn, "${module.metadata_s3_bucket.bucket.arn}/*"] + } + + statement { + sid = "DenyNonFullControlObjects" + effect = "Deny" + actions = ["s3:PutObject"] + resources = ["${module.metadata_s3_bucket.bucket.arn}/*"] + + principals { + identifiers = ["*"] + type = "AWS" + } + + condition { + test = "StringNotEquals" + variable = "s3:x-amz-acl" + + values = [ + "bucket-owner-full-control" + ] + } + } + +} + +data "aws_iam_policy_document" "logs_s3_bucket_policy_document" { + statement { + sid = "AllowPutFromCiUser" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/cicd-member-user"] + } + + actions = ["s3:PutObject", "s3:ListBucket"] + + resources = [module.logs_s3_bucket.bucket.arn, "${module.logs_s3_bucket.bucket.arn}/*"] + } + + statement { + sid = "DenyNonFullControlObjects" + effect = "Deny" + actions = ["s3:PutObject"] + resources = ["${module.logs_s3_bucket.bucket.arn}/*"] + + principals { + identifiers = ["*"] + type = "AWS" + } + condition { test = "StringNotEquals" variable = "s3:x-amz-acl" @@ -261,35 +472,27 @@ data "aws_iam_policy_document" "data_platform_product_bucket_policy_document" { # api gateway create data product metdata permissions data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json] + statement { - sid = "GetPutDataObject" + sid = "GetPutMetadata" effect = "Allow" actions = ["s3:GetObject", "s3:PutObject"] resources = [ - "${module.s3-bucket.bucket.arn}/metadata/*", - "${module.s3-bucket.bucket.arn}/data_product_metadata_spec/*" + "${module.metadata_s3_bucket.bucket.arn}/metadata/*", + "${module.metadata_s3_bucket.bucket.arn}/data_product_metadata_spec/*" ] } statement { - sid = "s3LogAccess" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - ] + sid = "ListBucket" + effect = "Allow" + actions = ["s3:ListBucket"] resources = [ - "${module.s3-bucket.bucket.arn}/logs/*" + module.metadata_s3_bucket.bucket.arn ] } - statement { - sid = "ListBucket" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [module.s3-bucket.bucket.arn, "${module.s3-bucket.bucket.arn}/*"] - } - statement { sid = "AllowLambdaToCreateLogGroup" effect = "Allow" @@ -314,11 +517,13 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda" } data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + statement { sid = "ListBucket" effect = "Allow" actions = ["s3:ListBucket"] - resources = [module.s3-bucket.bucket.arn, "${module.s3-bucket.bucket.arn}/*"] + resources = [module.data_s3_bucket.bucket.arn] } statement { sid = "InvokeAthenaLoadLambda" @@ -337,17 +542,6 @@ data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lamb "*" ] } - statement { - sid = "s3LogAccess" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - ] - resources = [ - "${module.s3-bucket.bucket.arn}/logs/*" - ] - } statement { sid = "LambdaLogGroup" effect = "Allow" @@ -357,11 +551,15 @@ data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lamb } data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files_lambda" { + source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + statement { - sid = "ListBucket" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [module.s3-bucket.bucket.arn, "${module.s3-bucket.bucket.arn}/*"] + sid = "ListBucket" + effect = "Allow" + actions = ["s3:ListBucket"] + resources = [ + module.data_s3_bucket.bucket.arn + ] } statement { sid = "InvokeAthenaLoadLambda" @@ -369,17 +567,7 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files actions = ["lambda:InvokeFunction"] resources = [module.data_product_athena_load_lambda.lambda_function_arn] } - statement { - sid = "s3LogAccess" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - ] - resources = [ - "${module.s3-bucket.bucket.arn}/logs/*" - ] - } + statement { sid = "LambdaLogGroup" effect = "Allow" diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 15373149367..d78fe842e3b 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -1,4 +1,3 @@ - module "data_product_docs_lambda" { source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 application_name = "data_product_docs" @@ -6,6 +5,7 @@ module "data_product_docs_lambda" { description = "Lambda for swagger api docs" function_name = "data_product_docs_${local.environment}" role_name = "docs_lambda_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_docs_lambda.json create_role = true reserved_concurrent_executions = 1 @@ -33,6 +33,7 @@ module "data_product_authorizer_lambda" { tags = local.tags description = "Lambda for custom API Gateway authorizer" role_name = "authorizer_lambda_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_authorizer_lambda.json function_name = "data_product_authorizer_${local.environment}" create_role = true @@ -43,10 +44,10 @@ module "data_product_authorizer_lambda" { tracing_mode = "Active" memory_size = 512 - environment_variables = { + environment_variables = merge(local.logger_environment_vars, { authorizationToken = "placeholder" api_resource_arn = "${aws_api_gateway_rest_api.data_platform.execution_arn}/*/*" - } + }) allowed_triggers = { @@ -66,6 +67,7 @@ module "data_product_get_glue_metadata_lambda" { tags = local.tags description = "Lambda to retrieve Glue metadata for a specified table in a database" role_name = "get_glue_metadata_lambda_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_get_glue_metadata_lambda.json function_name = "data_product_get_glue_metadata_${local.environment}" create_role = true @@ -88,12 +90,44 @@ module "data_product_get_glue_metadata_lambda" { } +module "data_product_landing_to_raw_lambda" { + source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 + application_name = "data_product_landing_to_raw" + tags = local.tags + description = "Lambda to retrieve Glue metadata for a specified table in a database" + role_name = "landing_to_raw_lambda_role_${local.environment}" + policy_json_attached = true + policy_json = data.aws_iam_policy_document.landing_to_raw_lambda_policy.json + function_name = "data_product_landing_to_raw_${local.environment}" + create_role = true + reserved_concurrent_executions = 1 + + image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-landing-to-raw-lambda-ecr-repo:${local.landing_to_raw_version}" + timeout = 600 + tracing_mode = "Active" + memory_size = 512 + + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) + + allowed_triggers = { + + AllowExecutionFromCloudWatch = { + action = "lambda:InvokeFunction" + function_name = "data_product_landing_to_raw_${local.environment}" + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.object_created_data_landing.arn + } + } + +} + module "data_product_presigned_url_lambda" { source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 application_name = "data_product_presigned_url" tags = local.tags description = "Lambda to generate a presigned url for uploading data" role_name = "presigned_url_lambda_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_presigned_url_lambda.json function_name = "data_product_presigned_url_${local.environment}" create_role = true @@ -104,13 +138,7 @@ module "data_product_presigned_url_lambda" { tracing_mode = "Active" memory_size = 512 - environment_variables = { - RAW_DATA_BUCKET = module.s3-bucket.bucket.id - CURATED_DATA_BUCKET = module.s3-bucket.bucket.id - LOG_BUCKET = module.s3-bucket.bucket.id - METADATA_BUCKET = module.s3-bucket.bucket.id - LANDING_ZONE_BUCKET = module.s3-bucket.bucket.id - } + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) allowed_triggers = { @@ -130,6 +158,7 @@ module "data_product_athena_load_lambda" { tags = local.tags description = "Lambda to load and transform raw data products landing in s3. Creates partitioned parquet tables" role_name = "athena_load_lambda_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.athena_load_lambda_function_policy.json function_name = "data_product_athena_load_${local.environment}" create_role = true @@ -140,14 +169,9 @@ module "data_product_athena_load_lambda" { tracing_mode = "Active" memory_size = 512 - environment_variables = { - ENVIRONMENT = local.environment - RAW_DATA_BUCKET = module.s3-bucket.bucket.id - CURATED_DATA_BUCKET = module.s3-bucket.bucket.id - LOG_BUCKET = module.s3-bucket.bucket.id - METADATA_BUCKET = module.s3-bucket.bucket.id - LANDING_ZONE_BUCKET = module.s3-bucket.bucket.id - } + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { + ENVIRONMENT = local.environment + }) allowed_triggers = { @@ -168,6 +192,7 @@ module "data_product_create_metadata_lambda" { tags = local.tags description = "Lambda to create the first version of a json metadata file for a data product" role_name = "data_product_metadata_lambda_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_create_metadata_lambda.json function_name = "data_product_create_metadata_${local.environment}" create_role = true @@ -178,10 +203,9 @@ module "data_product_create_metadata_lambda" { tracing_mode = "Active" memory_size = 128 - environment_variables = { + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { ENVIRONMENT = local.environment - BUCKET_NAME = module.s3-bucket.bucket.id - } + }) allowed_triggers = { @@ -201,6 +225,7 @@ module "reload_data_product_lambda" { tags = local.tags description = "Reload the data in a data product from raw history to curated, and recreate the athena tables." role_name = "reload_data_product_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_reload_data_product_lambda.json function_name = "reload_data_product_${local.environment}" create_role = true @@ -211,14 +236,9 @@ module "reload_data_product_lambda" { tracing_mode = "Active" memory_size = 512 - environment_variables = { - RAW_DATA_BUCKET = module.s3-bucket.bucket.id - CURATED_DATA_BUCKET = module.s3-bucket.bucket.id - LOG_BUCKET = module.s3-bucket.bucket.id - METADATA_BUCKET = module.s3-bucket.bucket.id - LANDING_ZONE_BUCKET = module.s3-bucket.bucket.id - ATHENA_LOAD_LAMBDA = module.data_product_athena_load_lambda.lambda_function_name - } + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { + ATHENA_LOAD_LAMBDA = module.data_product_athena_load_lambda.lambda_function_name + }) } @@ -228,6 +248,7 @@ module "resync_unprocessed_files_lambda" { tags = local.tags description = "Retrigger the athena load for extraction timestamps in raw history and not in curated data, for one data product" role_name = "resync_unprocessed_files_role_${local.environment}" + policy_json_attached = true policy_json = data.aws_iam_policy_document.iam_policy_document_for_resync_unprocessed_files_lambda.json function_name = "resync_unprocessed_files_${local.environment}" create_role = true @@ -238,10 +259,8 @@ module "resync_unprocessed_files_lambda" { tracing_mode = "Active" memory_size = 512 - environment_variables = { - RAW_DATA_BUCKET = module.s3-bucket.bucket.id - CURATED_DATA_BUCKET = module.s3-bucket.bucket.id - ATHENA_LOAD_LAMBDA = module.data_product_athena_load_lambda.lambda_function_name - } + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { + ATHENA_LOAD_LAMBDA = module.data_product_athena_load_lambda.lambda_function_name + }) } diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf index 25b56e6ac0a..66aa77b6b7b 100644 --- a/terraform/environments/data-platform/locals.tf +++ b/terraform/environments/data-platform/locals.tf @@ -32,4 +32,17 @@ locals { create_metadata_version = lookup(var.create_metadata_versions, local.environment) resync_unprocessed_files_version = lookup(var.resync_unprocessed_files_versions, local.environment) reload_data_product_version = lookup(var.reload_data_product_versions, local.environment) + landing_to_raw_version = lookup(var.landing_to_raw_versions, local.environment) + + # Environment vars that are used by many lambdas + logger_environment_vars = { + LOG_BUCKET = module.logs_s3_bucket.bucket.id + } + + storage_environment_vars = { + RAW_DATA_BUCKET = module.data_s3_bucket.bucket.id + CURATED_DATA_BUCKET = module.data_s3_bucket.bucket.id + METADATA_BUCKET = module.metadata_s3_bucket.bucket.id + LANDING_ZONE_BUCKET = module.data_landing_s3_bucket.bucket.id + } } diff --git a/terraform/environments/data-platform/s3.tf b/terraform/environments/data-platform/s3.tf index cd6f86bab15..395e0bede69 100644 --- a/terraform/environments/data-platform/s3.tf +++ b/terraform/environments/data-platform/s3.tf @@ -1,9 +1,9 @@ - +# TO BE REMOVED module "s3-bucket" { #tfsec:ignore:aws-s3-enable-versioning source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" bucket_prefix = "data-platform-products-${local.environment}" - versioning_enabled = true + versioning_enabled = false # Refer to the below section "Replication" before enabling replication replication_enabled = false bucket_policy = [data.aws_iam_policy_document.data_platform_product_bucket_policy_document.json] @@ -58,6 +58,193 @@ module "s3-bucket" { #tfsec:ignore:aws-s3-enable-versioning tags = local.tags } +module "data_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" + + bucket_prefix = "data-${local.environment}" + versioning_enabled = false + # Refer to the below section "Replication" before enabling replication + replication_enabled = false + bucket_policy = [data.aws_iam_policy_document.data_s3_bucket_policy_document.json] + providers = { + # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the + # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation + # Platform team to add a new provider for the additional Region. + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 730 + } + } + ] + + tags = local.tags +} + + +module "metadata_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" + + bucket_prefix = "metadata-${local.environment}" + versioning_enabled = false + # Refer to the below section "Replication" before enabling replication + replication_enabled = false + bucket_policy = [data.aws_iam_policy_document.metadata_s3_bucket_policy_document.json] + providers = { + # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the + # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation + # Platform team to add a new provider for the additional Region. + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 730 + } + + } + ] + + tags = local.tags +} + + +module "logs_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" + + bucket_prefix = "logs-${local.environment}" + versioning_enabled = false + # Refer to the below section "Replication" before enabling replication + replication_enabled = false + bucket_policy = [data.aws_iam_policy_document.logs_s3_bucket_policy_document.json] + providers = { + # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the + # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation + # Platform team to add a new provider for the additional Region. + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 730 + } + + } + ] + + tags = local.tags +} + + + +module "data_landing_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" + + bucket_prefix = "data-landing-${local.environment}" + versioning_enabled = false + # Refer to the below section "Replication" before enabling replication + replication_enabled = false + bucket_policy = [data.aws_iam_policy_document.data_landing_s3_bucket_policy_document.json] + providers = { + # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the + # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation + # Platform team to add a new provider for the additional Region. + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 730 + } + + } + ] + + tags = local.tags +} + module "s3_athena_query_results_bucket" { #tfsec:ignore:aws-s3-enable-versioning source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" @@ -119,13 +306,18 @@ module "s3_athena_query_results_bucket" { #tfsec:ignore:aws-s3-enable-versioning } resource "aws_s3_bucket_notification" "bucket_notification" { - bucket = module.s3-bucket.bucket.id + bucket = module.data_s3_bucket.bucket.id + eventbridge = true +} + +resource "aws_s3_bucket_notification" "landing_bucket_notification" { + bucket = module.data_landing_s3_bucket.bucket.id eventbridge = true } # load the json schema for data product metadata resource "aws_s3_object" "object" { - bucket = module.s3-bucket.bucket.id + bucket = module.metadata_s3_bucket.bucket.id key = "data_product_metadata_spec/v1.1.0/moj_data_product_metadata_spec.json" source = "data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json" etag = filemd5("data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json") diff --git a/terraform/environments/data-platform/triggers.tf b/terraform/environments/data-platform/triggers.tf index 7dbd6da964a..883107f6d39 100644 --- a/terraform/environments/data-platform/triggers.tf +++ b/terraform/environments/data-platform/triggers.tf @@ -6,9 +6,24 @@ resource "aws_cloudwatch_event_rule" "object_created_raw_data" { "source" : ["aws.s3"], "detail-type" : ["Object Created"], "detail" : { - "bucket" : { "name" : [module.s3-bucket.bucket.id] }, + "bucket" : { "name" : [module.data_s3_bucket.bucket.id] }, "object" : { - "key" : [{ "prefix" : "raw_data/" }] + "key" : [{ "prefix" : "raw/" }] + } + } + }) +} + +resource "aws_cloudwatch_event_rule" "object_created_data_landing" { + name = "object_created_data_landing" + tags = local.tags + event_pattern = jsonencode({ + "source" : ["aws.s3"], + "detail-type" : ["Object Created"], + "detail" : { + "bucket" : { "name" : [module.data_landing_s3_bucket.bucket.id] }, + "object" : { + "key" : [{ "prefix" : "landing/" }] } } }) @@ -19,3 +34,9 @@ resource "aws_cloudwatch_event_target" "athena_load_lambda_trigger" { target_id = "athena" arn = module.data_product_athena_load_lambda.lambda_function_arn } + +resource "aws_cloudwatch_event_target" "object_created_data_landing" { + rule = aws_cloudwatch_event_rule.object_created_data_landing.name + target_id = "landing_to_raw" + arn = module.data_product_landing_to_raw_lambda.lambda_function_arn +} diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf index 8fed439c911..7c92f7fc7b2 100644 --- a/terraform/environments/data-platform/variables.tf +++ b/terraform/environments/data-platform/variables.tf @@ -29,3 +29,7 @@ variable "resync_unprocessed_files_versions" { variable "reload_data_product_versions" { type = map(any) } + +variable "landing_to_raw_versions" { + type = map(any) +} diff --git a/terraform/modules/baseline_presets/s3.tf b/terraform/modules/baseline_presets/s3.tf index 5d1aa235778..10701d3abeb 100644 --- a/terraform/modules/baseline_presets/s3.tf +++ b/terraform/modules/baseline_presets/s3.tf @@ -109,6 +109,7 @@ locals { "s3:PutObject", "s3:PutObjectAcl", "s3:PutObjectTagging", + "s3:RestoreObject", ] principals = { type = "AWS" @@ -142,6 +143,7 @@ locals { "s3:PutObject", "s3:PutObjectAcl", "s3:PutObjectTagging", + "s3:RestoreObject", ] principals = { type = "AWS" @@ -161,6 +163,7 @@ locals { "s3:PutObjectAcl", "s3:PutObjectTagging", "s3:DeleteObject", + "s3:RestoreObject", ] principals = { type = "AWS" @@ -196,6 +199,7 @@ locals { "s3:PutObjectTagging", "s3:DeleteObject", "s3:DeleteObjectVersion", + "s3:RestoreObject", ] principals = { type = "AWS" @@ -227,6 +231,7 @@ locals { "s3:PutObject", "s3:PutObjectAcl", "s3:PutObjectTagging", + "s3:RestoreObject", ] } ] @@ -241,6 +246,7 @@ locals { "s3:PutObjectAcl", "s3:PutObjectTagging", "s3:DeleteObject", + "s3:RestoreObject", ] } ]