[DPR2-1576] S3 bucket expiration override

terraform/environments/digital-prison-reporting/application_variables.json

@@ -181,7 +181,7 @@
       "enable_dbt_k8s_secrets": true,
       "dpr_generic_athena_workgroup": true,
       "analytics_generic_athena_workgroup": true,
-      "redshift_table_expiry_seconds": "604800",
+      "redshift_table_expiry_days": 7,
       "enable_s3_data_migrate_lambda": true
     },
     "test": {
@@ -364,7 +364,7 @@
       "enable_dbt_k8s_secrets": true,
       "dpr_generic_athena_workgroup": true,
       "analytics_generic_athena_workgroup": true,
-      "redshift_table_expiry_seconds": "604800",
+      "redshift_table_expiry_days": 7,
       "enable_s3_data_migrate_lambda": true
     },
     "preproduction": {
@@ -567,7 +567,7 @@
           ]
         }
       ],
-      "redshift_table_expiry_seconds": "604800",
+      "redshift_table_expiry_days": 7,
       "enable_s3_data_migrate_lambda": true
     },
     "production": {
@@ -765,7 +765,7 @@
           ]
         }
       ],
-      "redshift_table_expiry_seconds": "86400",
+      "redshift_table_expiry_days": 1,
       "enable_s3_data_migrate_lambda": false
     }
   }

terraform/environments/digital-prison-reporting/locals.tf

@@ -240,10 +240,12 @@ locals {
   lambda_redshift_table_expiry_cluster_id          = module.datamart.cluster_id
   lambda_redshift_table_expiry_database_name       = module.datamart.cluster_database_name
   lambda_redshift_table_expiry_schedule_expression = "rate(1 hour)"
-  lambda_redshift_table_expiry_seconds             = local.application_data.accounts[local.environment].redshift_table_expiry_seconds
+  lambda_redshift_table_expiry_seconds             = (local.application_data.accounts[local.environment].redshift_table_expiry_days * 86400)
   lambda_redshift_table_expiry_timeout_seconds     = 900
   lambda_redshift_table_expiry_memory_size         = 1024
 
+  s3_redshift_table_expiry_days                    = local.application_data.accounts[local.environment].redshift_table_expiry_days + 1
+
   reporting_lambda_code_s3_key = "build-artifacts/digital-prison-reporting-lambdas/jars/digital-prison-reporting-lambdas-vLatest-all.jar"
 
   # s3 transfer

terraform/environments/digital-prison-reporting/main.tf

@@ -995,16 +995,17 @@ module "s3_working_bucket" {
   custom_kms_key              = local.s3_kms_arn
   create_notification_queue   = false # For SQS Queue
   enable_lifecycle            = true
-  enable_lifecycle_expiration = true
   lifecycle_category          = "long_term"
 
   override_expiration_rules = [
     {
-      prefix = "reports"
-      days   = 7
+      id = "reports"
+      prefix = "reports/"
+      days   = local.s3_redshift_table_expiry_days
     },
     {
-      prefix = "dpr"
+      id = "dpr"
+      prefix = "dpr/"
       days   = 7
     }
   ]

terraform/environments/digital-prison-reporting/modules/s3_bucket/main.tf

@@ -32,6 +32,8 @@ resource "aws_s3_bucket_public_access_block" "storage" {
 }
 
 resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" {
+  #checkov:skip=CKV_AWS_300:TODO Will be addressed as part of https://dsdmoj.atlassian.net/browse/DPR2-1083
+
   # Create the lifecycle configuration if either lifecycle or Intelligent-Tiering is enabled
   count = var.enable_lifecycle || var.enable_intelligent_tiering ? 1 : 0
 
@@ -102,6 +104,23 @@ resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" {
       storage_class = "INTELLIGENT_TIERING"
     }
   }
+
+  # Expiration rules
+  dynamic "rule" {
+    for_each = var.override_expiration_rules
+    content {
+      id     = rule.value.id
+      status = "Enabled"
+
+      filter {
+        prefix = rule.value.prefix
+      }
+
+      expiration {
+        days = rule.value.days
+      }
+    }
+  }
 }
 
 resource "aws_s3_bucket_server_side_encryption_configuration" "encryption" {

terraform/environments/digital-prison-reporting/modules/s3_bucket/variables.tf

@@ -1,6 +1,7 @@
 
 variable "name" {
   description = "Name of the Bucket"
+  type        = string
   default     = ""
 }
 
@@ -24,11 +25,13 @@ variable "cloudtrail_access_policy" {
 
 variable "s3_notification_name" {
   description = "S3 Notification Event Name"
+  type        = string
   default     = "s3-notification-event"
 }
 
 variable "create_s3" {
   description = "Setup S3 Buckets"
+  type        = bool
   default     = false
 }
 
@@ -40,60 +43,46 @@ variable "custom_kms_key" {
 
 variable "create_notification_queue" {
   description = "Setup Notification Queue"
+  type        = bool
   default     = false
 }
 
 variable "sqs_msg_retention_seconds" {
   description = "SQS Message Retention"
+  type        = number
   default     = 86400
 }
 
 variable "filter_prefix" {
   description = "S3 Notification Filter Prefix"
+  type        = string
   default     = null
 }
 
 variable "enable_lifecycle" {
   description = "Enabled Lifecycle for S3 Storage, Default is False"
+  type        = bool
   default     = false
 }
 
-#variable "expiration_days" {
-#  description = "Days to wait before deleting expired items."
-#  default     = 90
-#}
-
-#variable "expiration_prefix_redshift" {
-#  description = "Directory Prefix where Redshift Async query results are stored to apply expiration to."
-#  default     = "/"
-#}
-
-#variable "expiration_prefix_athena" {
-#  description = "Directory Prefix where Athena Async query results are stored to apply expiration to."
-#  default     = "/"
-#}
-
 variable "enable_versioning_config" {
   description = "Enable Versioning Config for S3 Storage, Default is Disabled"
+  type        = string
   default     = "Disabled"
 }
 
 variable "enable_s3_versioning" {
   description = "Enable Versioning for S3 Bucket, Default is false"
+  type        = bool
   default     = false
 }
 
 variable "enable_notification" {
   description = "Enable S3 Bucket Notifications, Default is false"
+  type        = bool
   default     = false
 }
 
-#variable "bucket_notifications" {
-#  type        = map(any)
-#  description = "AWS S3 Bucket Notifications"
-#  default     = null
-#}
-
 variable "bucket_notifications" {
   type        = any
   description = "AWS S3 Bucket Notifications"
@@ -112,12 +101,13 @@ variable "dependency_lambda" {
 
 variable "bucket_key" {
   description = "If Bucket Key is Enabled or Disabled"
+  type        = bool
   default     = true
 }
 
 ## Dynamic override_expiration_rules
 variable "override_expiration_rules" {
-  type    = list(object({ prefix = string, days = number }))
+  type    = list(object({ id = string, prefix = string, days = number }))
   default = []
 }
 
@@ -126,11 +116,6 @@ variable "lifecycle_category" {
   default = "standard" # Options: "short_term", "long_term", "temporary", "standard"
 }
 
-variable "enable_lifecycle_expiration" {
-  description = "Enable item expiration - requires 'enable_lifecycle' and 'override_expiration_rules' to be defined/enabled."
-  default     = false
-}
-
 variable "enable_intelligent_tiering" {
   description = "Enable Intelligent-Tiering storage class for S3 bucket"
   type        = bool