DPR2-151: Configure maintenance jobs and streaming job with data stor…

…age service retries
ministryofjustice · Oct 13, 2023 · fda2cef · fda2cef
1 parent 8b00859
commit fda2cef
Show file tree

Hide file tree

Showing 4 changed files with 124 additions and 64 deletions.
diff --git a/terraform/environments/digital-prison-reporting/application_variables.json b/terraform/environments/digital-prison-reporting/application_variables.json
@@ -14,9 +14,15 @@
       "reporting_hub_batch_duration_seconds": 30,
       "reporting_hub_add_idle_time_between_reads": true,
       "reporting_hub_idle_time_between_reads_in_millis": 140,
+      "reporting_hub_retry_max_attempts": 10,
+      "reporting_hub_retry_min_wait_millis": 100,
+      "reporting_hub_retry_max_wait_millis": 10000,
       "refresh_job_worker_type": "G.1X",
       "refresh_job_num_workers": 2,
       "refresh_job_log_level": "INFO",
+      "maintenance_job_retry_max_attempts": 10,
+      "maintenance_job_retry_min_wait_millis": 1000,
+      "maintenance_job_retry_max_wait_millis": 20000,
       "compact_raw_job_worker_type": "G.1X",
       "compact_raw_job_num_workers": 2,
       "compact_raw_job_log_level": "INFO",
@@ -91,9 +97,15 @@
       "reporting_hub_batch_duration_seconds": 30,
       "reporting_hub_add_idle_time_between_reads": true,
       "reporting_hub_idle_time_between_reads_in_millis": 140,
+      "reporting_hub_retry_max_attempts": 10,
+      "reporting_hub_retry_min_wait_millis": 100,
+      "reporting_hub_retry_max_wait_millis": 10000,
       "refresh_job_worker_type": "G.1X",
       "refresh_job_num_workers": 2,
       "refresh_job_log_level": "INFO",
+      "maintenance_job_retry_max_attempts": 10,
+      "maintenance_job_retry_min_wait_millis": 1000,
+      "maintenance_job_retry_max_wait_millis": 20000,
       "compact_raw_job_worker_type": "G.1X",
       "compact_raw_job_num_workers": 2,
       "compact_raw_job_log_level": "INFO",
@@ -168,9 +180,15 @@
       "reporting_hub_batch_duration_seconds": 30,
       "reporting_hub_add_idle_time_between_reads": false,
       "reporting_hub_idle_time_between_reads_in_millis": 140,
+      "reporting_hub_retry_max_attempts": 10,
+      "reporting_hub_retry_min_wait_millis": 100,
+      "reporting_hub_retry_max_wait_millis": 10000,
       "refresh_job_worker_type": "G.1X",
       "refresh_job_num_workers": 2,
       "refresh_job_log_level": "INFO",
+      "maintenance_job_retry_max_attempts": 10,
+      "maintenance_job_retry_min_wait_millis": 1000,
+      "maintenance_job_retry_max_wait_millis": 20000,
       "compact_raw_job_worker_type": "G.1X",
       "compact_raw_job_num_workers": 2,
       "compact_raw_job_log_level": "INFO",
@@ -245,9 +263,15 @@
       "reporting_hub_batch_duration_seconds": 40,
       "reporting_hub_add_idle_time_between_reads": false,
       "reporting_hub_idle_time_between_reads_in_millis": 140,
+      "reporting_hub_retry_max_attempts": 10,
+      "reporting_hub_retry_min_wait_millis": 100,
+      "reporting_hub_retry_max_wait_millis": 10000,
       "refresh_job_worker_type": "G.1X",
       "refresh_job_num_workers": 2,
       "refresh_job_log_level": "INFO",
+      "maintenance_job_retry_max_attempts": 10,
+      "maintenance_job_retry_min_wait_millis": 1000,
+      "maintenance_job_retry_max_wait_millis": 20000,
       "compact_raw_job_worker_type": "G.1X",
       "compact_raw_job_num_workers": 2,
       "compact_raw_job_log_level": "INFO",

diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf
@@ -58,11 +58,20 @@ locals {
 
   reporting_hub_idle_time_between_reads_in_millis = local.application_data.accounts[local.environment].reporting_hub_idle_time_between_reads_in_millis
 
+  reporting_hub_retry_max_attempts    = local.application_data.accounts[local.environment].reporting_hub_retry_max_attempts
+  reporting_hub_retry_min_wait_millis = local.application_data.accounts[local.environment].reporting_hub_retry_min_wait_millis
+  reporting_hub_retry_max_wait_millis = local.application_data.accounts[local.environment].reporting_hub_retry_max_wait_millis
+
   # Refresh Job
   refresh_job_worker_type = local.application_data.accounts[local.environment].refresh_job_worker_type
   refresh_job_num_workers = local.application_data.accounts[local.environment].refresh_job_num_workers
   refresh_job_log_level   = local.application_data.accounts[local.environment].refresh_job_log_level
 
+  # Common Maintenance Job settings
+  maintenance_job_retry_max_attempts    = local.application_data.accounts[local.environment].maintenance_job_retry_max_attempts
+  maintenance_job_retry_min_wait_millis = local.application_data.accounts[local.environment].maintenance_job_retry_min_wait_millis
+  maintenance_job_retry_max_wait_millis = local.application_data.accounts[local.environment].maintenance_job_retry_max_wait_millis
+
   # Compact Raw Job
   compact_raw_job_worker_type = local.application_data.accounts[local.environment].compact_raw_job_worker_type
   compact_raw_job_num_workers = local.application_data.accounts[local.environment].compact_raw_job_num_workers

diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf
@@ -44,30 +44,33 @@ module "glue_reporting_hub_job" {
   )
 
   arguments = {
-    "--extra-jars"                         = local.glue_jobs_latest_jar_location
-    "--job-bookmark-option"                = "job-bookmark-disable"
-    "--class"                              = "uk.gov.justice.digital.job.DataHubJob"
-    "--dpr.kinesis.stream.arn"             = module.kinesis_stream_ingestor.kinesis_stream_arn
-    "--dpr.aws.region"                     = local.account_region
-    "--dpr.curated.s3.path"                = "s3://${module.s3_curated_bucket.bucket_id}/"
-    "--dpr.batchDurationSeconds"           = local.reporting_hub_batch_duration_seconds
-    "--dpr.add.idle.time.between.reads"    = local.reporting_hub_add_idle_time_between_reads
-    "--dpr.idle.time.between.reads.millis" = local.reporting_hub_idle_time_between_reads_in_millis
-    "--dpr.raw.s3.path"                    = "s3://${module.s3_raw_bucket.bucket_id}/"
-    "--dpr.structured.s3.path"             = "s3://${module.s3_structured_bucket.bucket_id}/"
-    "--dpr.violations.s3.path"             = "s3://${module.s3_violation_bucket.bucket_id}/"
-    "--enable-metrics"                     = true
-    "--enable-spark-ui"                    = false
-    "--enable-auto-scaling"                = true
-    "--enable-job-insights"                = true
-    "--dpr.aws.dynamodb.endpointUrl"       = "https://dynamodb.${local.account_region}.amazonaws.com"
-    "--dpr.contract.registryName"          = trimprefix(module.glue_registry_avro.registry_name, "${local.glue_avro_registry[0]}/")
-    "--dpr.domain.registry"                = "${local.project}-domain-registry-${local.environment}"
-    "--dpr.domain.target.path"             = "s3://${module.s3_domain_bucket.bucket_id}"
-    "--dpr.domain.catalog.db"              = module.glue_data_domain_database.db_name
-    "--dpr.redshift.secrets.name"          = "${local.project}-redshift-secret-${local.environment}"
-    "--dpr.datamart.db.name"               = "datamart"
-    "--dpr.log.level"                      = local.reporting_hub_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--job-bookmark-option"                 = "job-bookmark-disable"
+    "--class"                               = "uk.gov.justice.digital.job.DataHubJob"
+    "--dpr.kinesis.stream.arn"              = module.kinesis_stream_ingestor.kinesis_stream_arn
+    "--dpr.aws.region"                      = local.account_region
+    "--dpr.curated.s3.path"                 = "s3://${module.s3_curated_bucket.bucket_id}/"
+    "--dpr.batchDurationSeconds"            = local.reporting_hub_batch_duration_seconds
+    "--dpr.add.idle.time.between.reads"     = local.reporting_hub_add_idle_time_between_reads
+    "--dpr.idle.time.between.reads.millis"  = local.reporting_hub_idle_time_between_reads_in_millis
+    "--dpr.datastorage.retry.maxAttempts"   = local.reporting_hub_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.reporting_hub_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.reporting_hub_retry_max_wait_millis
+    "--dpr.raw.s3.path"                     = "s3://${module.s3_raw_bucket.bucket_id}/"
+    "--dpr.structured.s3.path"              = "s3://${module.s3_structured_bucket.bucket_id}/"
+    "--dpr.violations.s3.path"              = "s3://${module.s3_violation_bucket.bucket_id}/"
+    "--enable-metrics"                      = true
+    "--enable-spark-ui"                     = false
+    "--enable-auto-scaling"                 = true
+    "--enable-job-insights"                 = true
+    "--dpr.aws.dynamodb.endpointUrl"        = "https://dynamodb.${local.account_region}.amazonaws.com"
+    "--dpr.contract.registryName"           = trimprefix(module.glue_registry_avro.registry_name, "${local.glue_avro_registry[0]}/")
+    "--dpr.domain.registry"                 = "${local.project}-domain-registry-${local.environment}"
+    "--dpr.domain.target.path"              = "s3://${module.s3_domain_bucket.bucket_id}"
+    "--dpr.domain.catalog.db"               = module.glue_data_domain_database.db_name
+    "--dpr.redshift.secrets.name"           = "${local.project}-redshift-secret-${local.environment}"
+    "--dpr.datamart.db.name"                = "datamart"
+    "--dpr.log.level"                       = local.reporting_hub_log_level
   }
 }
 

diff --git a/terraform/environments/digital-prison-reporting/maintenance_jobs.tf b/terraform/environments/digital-prison-reporting/maintenance_jobs.tf
@@ -61,11 +61,14 @@ module "glue_compact_raw_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.compact_job_class
-    "--dpr.maintenance.root.path" = local.raw_zone_nomis_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.compact_raw_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.compact_job_class
+    "--dpr.maintenance.root.path"           = local.raw_zone_nomis_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.compact_raw_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 # Glue Job, Compact Structured zone
@@ -102,11 +105,14 @@ module "glue_compact_structured_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.compact_job_class
-    "--dpr.maintenance.root.path" = local.structured_zone_nomis_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.compact_structured_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.compact_job_class
+    "--dpr.maintenance.root.path"           = local.structured_zone_nomis_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.compact_structured_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 # Glue Job, Compact Curated zone
@@ -143,11 +149,14 @@ module "glue_compact_curated_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.compact_job_class
-    "--dpr.maintenance.root.path" = local.curated_zone_nomis_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.compact_curated_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.compact_job_class
+    "--dpr.maintenance.root.path"           = local.curated_zone_nomis_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.compact_curated_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 # Glue Job, Compact Domain zone
@@ -184,11 +193,14 @@ module "glue_compact_domain_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.compact_job_class
-    "--dpr.maintenance.root.path" = local.domain_zone_root_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.compact_domain_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.compact_job_class
+    "--dpr.maintenance.root.path"           = local.domain_zone_root_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.compact_domain_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 
@@ -226,11 +238,14 @@ module "glue_retention_raw_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.retention_job_class
-    "--dpr.maintenance.root.path" = local.raw_zone_nomis_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.retention_raw_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.retention_job_class
+    "--dpr.maintenance.root.path"           = local.raw_zone_nomis_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.retention_raw_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 # Glue Job, Retention (vacuum) Structured zone
@@ -267,11 +282,14 @@ module "glue_retention_structured_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.retention_job_class
-    "--dpr.maintenance.root.path" = local.structured_zone_nomis_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.retention_structured_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.retention_job_class
+    "--dpr.maintenance.root.path"           = local.structured_zone_nomis_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.retention_structured_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 # Glue Job, Retention (vacuum) Curated zone
@@ -308,11 +326,14 @@ module "glue_retention_curated_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.retention_job_class
-    "--dpr.maintenance.root.path" = local.curated_zone_nomis_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.retention_curated_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.retention_job_class
+    "--dpr.maintenance.root.path"           = local.curated_zone_nomis_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.retention_curated_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }
 # Glue Job, Retention (vacuum) Domain zone
@@ -349,11 +370,14 @@ module "glue_retention_domain_job" {
   )
 
   arguments = {
-    "--extra-jars"                = local.glue_jobs_latest_jar_location
-    "--class"                     = local.retention_job_class
-    "--dpr.maintenance.root.path" = local.domain_zone_root_path
-    "--datalake-formats"          = "delta"
-    "--dpr.log.level"             = local.retention_domain_job_log_level
+    "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--class"                               = local.retention_job_class
+    "--dpr.maintenance.root.path"           = local.domain_zone_root_path
+    "--datalake-formats"                    = "delta"
+    "--dpr.log.level"                       = local.retention_domain_job_log_level
+    "--dpr.datastorage.retry.maxAttempts"   = local.maintenance_job_retry_max_attempts
+    "--dpr.datastorage.retry.minWaitMillis" = local.maintenance_job_retry_min_wait_millis
+    "--dpr.datastorage.retry.maxWaitMillis" = local.maintenance_job_retry_max_wait_millis
   }
 }