Merge branch 'main' into TM-756

ministryofjustice · Dec 4, 2024 · 91d52e3 · 91d52e3
2 parents 8e4fa4a + 9f41f14
commit 91d52e3
Show file tree

Hide file tree

Showing 122 changed files with 2,874 additions and 982 deletions.
diff --git a/.github/workflows/awsnuke.yml b/.github/workflows/awsnuke.yml
@@ -133,11 +133,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
     env:
       ACCOUNT_NAME: ${{ matrix.nuke_accts }}
@@ -217,11 +217,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
     env:
       AWS_ACCESS_KEY_ID:  ${{ secrets.TESTING_AWS_ACCESS_KEY_ID }}

diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml
@@ -38,7 +38,7 @@ jobs:
         run: tflint --disable-rule=terraform_unused_declarations --format sarif > tflint.sarif
       - name: Upload SARIF file
         if: success() || failure()
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: tflint.sarif
   trivy:
@@ -63,7 +63,7 @@ jobs:
 
       - name: Upload Trivy scan results to GitHub Security tab
         if: success() || failure()
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: 'trivy-results.sarif'
   checkov:
@@ -81,7 +81,7 @@ jobs:
           fetch-depth: 0
       - name: Run Checkov action
         id: checkov
-        uses: bridgecrewio/checkov-action@b8f970b660bc01f598fc2f108eabd9e8dee728f8 # v12.2924.0
+        uses: bridgecrewio/checkov-action@cb92fa252c0bd4612aa02aa4f0a6f405f2ccaa5c # v12.2927.0
         with:
           directory: ./
           framework: terraform
@@ -90,6 +90,6 @@ jobs:
           skip_check: CKV_GIT_1,CKV_AWS_126,CKV2_AWS_38,CKV2_AWS_39
       - name: Upload SARIF file
         if: success() || failure()
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: ./checkov.sarif
diff --git a/.github/workflows/nuke-redeploy.yml b/.github/workflows/nuke-redeploy.yml
@@ -93,11 +93,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
 
     env:

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: results.sarif
diff --git a/terraform/environments/analytical-platform-compute/ec2-instances.tf b/terraform/environments/analytical-platform-compute/ec2-instances.tf
@@ -0,0 +1,35 @@
+module "debug_instance" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/ec2-instance/aws"
+  version = "5.7.1"
+
+  name                        = "network-debug"
+  ami                         = "ami-0e8d228ad90af673b" # Ubuntu Server 24.04 LTS
+  instance_type               = "t3.micro"
+  subnet_id                   = element(module.vpc.private_subnets, 0)
+  vpc_security_group_ids      = [module.debug_instance_security_group.security_group_id]
+  associate_public_ip_address = false
+
+  root_block_device = [
+    {
+      encrypted   = true
+      volume_type = "gp3"
+      volume_size = 8
+    }
+  ]
+
+  create_iam_instance_profile = true
+  iam_role_policies = {
+    SSMCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  }
+
+  metadata_options = {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 1
+    http_tokens                 = "required"
+    instance_metadata_tags      = "enabled"
+  }
+
+  tags = local.tags
+}
diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf
@@ -43,6 +43,7 @@ locals {
 
       /* UI */
       ui_hostname = "development.analytical-platform.service.justice.gov.uk"
+
     }
     test = {
       /* VPC */
@@ -131,6 +132,11 @@ locals {
 
       /* UI */
       ui_hostname = "analytical-platform.service.justice.gov.uk"
+
+      /* LF Domain Tags */
+      cadet_lf_tags = {
+        domain = ["bold", "civil", "courts", "general", "criminal_history", "development_sandpit", "electronic_monitoring", "finance", "interventions", "opg", "performance", "risk", "people", "prison", "probation", "staging", "victims", "victims_case_management"] # extracted from bucket paths
+      }
     }
   }
 }
diff --git a/terraform/environments/analytical-platform-compute/iam-policies.tf b/terraform/environments/analytical-platform-compute/iam-policies.tf
@@ -426,11 +426,13 @@ data "aws_iam_policy_document" "copy_apdp_cadet_metadata_to_compute_policy" {
     sid    = "AlterLFTags"
     effect = "Allow"
     actions = [
+      "lakeformation:ListLFTags",
+      "lakeformation:GetLFTag",
+      "lakeformation:CreateLFTag",
+      "lakeformation:UpdateLFTag",
       "lakeformation:AddLFTagsToResource",
       "lakeformation:RemoveLFTagsFromResource",
       "lakeformation:GetResourceLFTags",
-      "lakeformation:ListLFTags",
-      "lakeformation:GetLFTag",
       "lakeformation:SearchTablesByLFTags",
       "lakeformation:SearchDatabasesByLFTags",
     ]

diff --git a/terraform/environments/analytical-platform-compute/iam-roles.tf b/terraform/environments/analytical-platform-compute/iam-roles.tf
@@ -381,16 +381,15 @@ module "copy_apdp_cadet_metadata_to_compute_assumable_role" {
   version = "5.48.0"
 
   allow_self_assume_role = false
-  trusted_role_arns      = ["arn:aws:iam::${local.environment_management.account_ids["analytical-platform-data-production"]}:role/create-a-derived-table"]
-  create_role            = true
-  role_requires_mfa      = false
-  role_name              = "copy-apdp-cadet-metadata-to-compute"
+  trusted_role_arns = [
+    "arn:aws:iam::${local.environment_management.account_ids["analytical-platform-data-production"]}:role/create-a-derived-table",
+    "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-reserved/sso.amazonaws.com/${data.aws_region.current.name}/${one(data.aws_iam_roles.data_engineering_sso_role.names)}",
+    "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-reserved/sso.amazonaws.com/${data.aws_region.current.name}/${one(data.aws_iam_roles.eks_sso_access_role.names)}",
+  ]
+  create_role       = true
+  role_requires_mfa = false
+  role_name         = "copy-apdp-cadet-metadata-to-compute"
 
   custom_role_policy_arns = [module.copy_apdp_cadet_metadata_to_compute_policy.arn]
   # number_of_custom_role_policy_arns = 1
 }
-
-moved {
-  from = module.analytical_platform_cadet_runner
-  to   = module.copy_apdp_cadet_metadata_to_compute_assumable_role
-}
diff --git a/terraform/environments/analytical-platform-compute/lakeformation-permissions.tf b/terraform/environments/analytical-platform-compute/lakeformation-permissions.tf
@@ -0,0 +1,58 @@
+
+resource "aws_lakeformation_lf_tag" "source" {
+  count  = terraform.workspace == "analytical-platform-compute-production" ? 1 : 0
+  key    = "source"
+  values = ["create-a-derived-table"]
+}
+
+resource "aws_lakeformation_permissions" "cadet_all_data" {
+  for_each = (terraform.workspace == "analytical-platform-compute-production" ?
+  toset(["TABLE", "DATABASE"]) : toset([]))
+
+  principal   = module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
+  permissions = ["ALL"] # https://docs.aws.amazon.com/lake-formation/latest/dg/lf-permissions-reference.html
+
+  lf_tag_policy {
+    resource_type = each.value
+    expression {
+      key    = "source"
+      values = ["create-a-derived-table"]
+    }
+  }
+}
+
+resource "aws_lakeformation_lf_tag" "domain" {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+  key      = each.key
+  values   = each.value
+}
+
+resource "aws_lakeformation_permissions" "cadet_domain_database_data" {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+
+  principal   = module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
+  permissions = ["ALL"] # https://docs.aws.amazon.com/lake-formation/latest/dg/lf-permissions-reference.html
+
+  lf_tag_policy {
+    resource_type = "DATABASE"
+    expression {
+      key    = each.key
+      values = each.value
+    }
+  }
+}
+
+resource "aws_lakeformation_permissions" "cadet_domain_table_data" {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+
+  principal   = module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
+  permissions = ["ALL"] # https://docs.aws.amazon.com/lake-formation/latest/dg/lf-permissions-reference.html
+
+  lf_tag_policy {
+    resource_type = "TABLE"
+    expression {
+      key    = each.key
+      values = each.value
+    }
+  }
+}
diff --git a/terraform/environments/analytical-platform-compute/security-groups.tf b/terraform/environments/analytical-platform-compute/security-groups.tf
@@ -55,3 +55,19 @@ module "quicksight_shared_vpc_security_group" {
 
   tags = local.tags
 }
+
+/* This security group is temporary and will be retired when we're satisfied with DataSync end-to-end */
+module "debug_instance_security_group" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+
+  source  = "terraform-aws-modules/security-group/aws"
+  version = "5.2.0"
+
+  name   = "debug-instance"
+  vpc_id = module.vpc.vpc_id
+
+  egress_cidr_blocks = ["0.0.0.0/0"]
+  egress_rules       = ["all-all"]
+
+  tags = local.tags
+}
diff --git a/...ents/analytical-platform-compute/src/helm/values/amazon-prometheus-proxy/values.yml.tftpl b/...ents/analytical-platform-compute/src/helm/values/amazon-prometheus-proxy/values.yml.tftpl
@@ -14,6 +14,10 @@ alertmanager:
 grafana:
   enabled: false
 
+kube-state-metrics:
+  extraArgs:
+    - --metric-labels-allowlist=pods=[*]
+
 prometheus:
   agentMode: true
   serviceAccount:

diff --git a/terraform/environments/ccms-ebs/application_variables.json b/terraform/environments/ccms-ebs/application_variables.json
@@ -370,7 +370,7 @@
       "ebs_iops_ebsdb_dbf03": 12000,
       "ebs_size_ebsdb_dbf03": 3000,
       "ebs_iops_ebsdb_dbf04": 28000,
-      "ebs_size_ebsdb_dbf04": 500,
+      "ebs_size_ebsdb_dbf04": 1000,
       "ebs_size_ebsdb_redoA": 100,
       "ebs_size_ebsdb_redoB": 50,
       "ebs_size_ebsdb_techst": 50,

diff --git a/terraform/environments/delius-core/files/user_outbound_table_mapping.json b/terraform/environments/delius-core/files/user_outbound_table_mapping.json
@@ -28,11 +28,11 @@
       "rule-name": "remove_staff_id",
       "rule-target": "column",
       "object-locator": {
-          "schema-name": "DELIUS_APP_SCHEMA",
-          "table-name": "USER_",
-          "column-name": "STAFF_ID"
+        "schema-name": "DELIUS_APP_SCHEMA",
+        "table-name": "USER_",
+        "column-name": "STAFF_ID"
       },
       "rule-action": "remove-column"
-  }
+    }
   ]
 }
diff --git a/terraform/environments/delius-core/locals_environments_all.tf b/terraform/environments/delius-core/locals_environments_all.tf
@@ -17,7 +17,7 @@ locals {
     ordered_subnets               = [local.ordered_subnet_ids]
     data_subnet_ids               = data.aws_subnets.shared-data.ids
     data_subnet_a_id              = data.aws_subnet.data_subnets_a.id
-    route53_inner_zone_info       = data.aws_route53_zone.inner
+    route53_inner_zone            = data.aws_route53_zone.inner
     route53_network_services_zone = data.aws_route53_zone.network-services
     route53_external_zone         = data.aws_route53_zone.external
     shared_vpc_id                 = data.aws_vpc.shared.id
@@ -26,7 +26,8 @@ locals {
       general_shared = data.aws_kms_key.general_shared.arn
       rds_shared     = data.aws_kms_key.rds_shared.arn
     }
-    dns_suffix = "${local.application_name}.${var.networking[0].business-unit}-${local.environment}.modernisation-platform.service.justice.gov.uk"
+    dns_suffix          = "${local.application_name}.${var.networking[0].business-unit}-${local.environment}.modernisation-platform.service.justice.gov.uk"
+    internal_dns_suffix = "${local.application_name}.${var.networking[0].business-unit}-${local.environment}.modernisation-platform.internal"
   }
 
   platform_vars = {

diff --git a/terraform/environments/delius-core/locals_preproduction.tf b/terraform/environments/delius-core/locals_preproduction.tf
@@ -21,13 +21,13 @@ locals {
     encrypted                   = true
     migration_source_account_id = "010587221707"
     migration_lambda_role       = "ldap-data-migration-lambda-role"
-    efs_throughput_mode         = "bursting"
+    efs_throughput_mode         = "elastic"
     efs_provisioned_throughput  = null
     efs_backup_schedule         = "cron(0 19 * * ? *)",
     efs_backup_retention_period = "30"
     port                        = 389
     tls_port                    = 636
-    desired_count               = 0
+    desired_count               = 1
   }