Merge pull request #3870 from ministryofjustice/dpl-1662-delete-table…

…-infra 🚧 Define delete table infrastructure
ministryofjustice · Nov 3, 2023 · 31bce88 · 31bce88
2 parents c3c43a6 + 0e2f73f
commit 31bce88
Show file tree

Hide file tree

Showing 8 changed files with 135 additions and 22 deletions.
diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf
@@ -326,6 +326,36 @@ resource "aws_api_gateway_integration" "update_schema_for_data_product_table_nam
   }
 }
 
+# /data-product/{data-product-name}/table/{table-name} DELETE method
+resource "aws_api_gateway_method" "delete_table_for_data_product" {
+  authorization = "CUSTOM"
+  authorizer_id = aws_api_gateway_authorizer.authorizer.id
+  http_method   = "DELETE"
+  resource_id   = aws_api_gateway_resource.data_product_table_name.id
+  rest_api_id   = aws_api_gateway_rest_api.data_platform.id
+
+  request_parameters = {
+    "method.request.header.Authorization"   = true,
+    "method.request.path.data-product-name" = true,
+    "method.request.path.table-name"        = true,
+  }
+}
+
+# /data-product/{data-product-name}/table/{table-name} (delete table and data) lambda integration
+resource "aws_api_gateway_integration" "delete_table_for_data_product_to_lambda" {
+  http_method             = aws_api_gateway_method.delete_table_for_data_product.http_method
+  resource_id             = aws_api_gateway_resource.data_product_table_name.id
+  rest_api_id             = aws_api_gateway_rest_api.data_platform.id
+  integration_http_method = "POST"
+  type                    = "AWS_PROXY"
+  uri                     = module.delete_table_for_data_product_lambda.lambda_function_invoke_arn
+
+  request_parameters = {
+    "integration.request.path.data-product-name" = "method.request.path.data-product-name",
+    "integration.request.path.table-name"        = "method.request.path.table-name",
+  }
+}
+
 # API docs endpoint
 
 resource "aws_api_gateway_resource" "docs" {

diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json
@@ -78,9 +78,15 @@
     "production": "1.0.1"
   },
   "preview_data_versions": {
-    "development": "1.0.1",
-    "test": "1.0.1",
-    "preproduction": "1.0.1",
-    "production": "1.0.1"
+    "development": "1.0.0",
+    "test": "1.0.0",
+    "preproduction": "1.0.0",
+    "production": "1.0.0"
+  },
+  "delete_table_for_data_product_versions": {
+    "development": "2.0.0",
+    "test": "2.0.0",
+    "preproduction": "2.0.0",
+    "production": "2.0.0"
   }
 }
diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf
@@ -659,3 +659,12 @@ data "aws_iam_policy_document" "iam_policy_document_for_preview_data" {
     ]
   }
 }
+
+data "aws_iam_policy_document" "iam_policy_document_for_delete_table_for_data_product_lambda" {
+  source_policy_documents = [
+    data.aws_iam_policy_document.log_to_bucket.json,
+    data.aws_iam_policy_document.read_metadata.json,
+    data.aws_iam_policy_document.write_metadata.json,
+    data.aws_iam_policy_document.create_write_lambda_logs.json,
+  ]
+}
diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf
@@ -402,4 +402,32 @@ module "preview_data_lambda" {
   }
 }
 
+module "delete_table_for_data_product_lambda" {
+  source                         = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1
+  application_name               = "delete_table"
+  tags                           = local.tags
+  description                    = "Delete table and data for a data product"
+  role_name                      = "delete_table_for_data_product_role_${local.environment}"
+  policy_json                    = data.aws_iam_policy_document.iam_policy_document_for_delete_table_for_data_product_lambda.json
+  policy_json_attached           = true
+  function_name                  = "delete_table_for_data_product_${local.environment}"
+  create_role                    = true
+  reserved_concurrent_executions = 1
+
+  image_uri    = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-delete-table-for-data-product-lambda-ecr-repo:${local.get_schema_version}"
+  timeout      = 600
+  tracing_mode = "Active"
+  memory_size  = 128
+
+  environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars)
+
+  allowed_triggers = {
+
+    AllowExecutionFromAPIGateway = {
+      action     = "lambda:InvokeFunction"
+      principal  = "apigateway.amazonaws.com"
+      source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.delete_table_for_data_product.http_method}${aws_api_gateway_resource.data_product_table_name.path}"
+    }
+  }
+}
 
diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf
@@ -26,20 +26,21 @@ locals {
   max_concurrent                   = 5
   glue_log_group_retention_in_days = 7
 
-  docs_version                     = lookup(var.docs_versions, local.environment)
-  authorizer_version               = lookup(var.authorizer_versions, local.environment)
-  get_glue_metadata_version        = lookup(var.get_glue_metadata_versions, local.environment)
-  presigned_url_version            = lookup(var.presigned_url_versions, local.environment)
-  athena_load_version              = lookup(var.athena_load_versions, local.environment)
-  create_metadata_version          = lookup(var.create_metadata_versions, local.environment)
-  resync_unprocessed_files_version = lookup(var.resync_unprocessed_files_versions, local.environment)
-  reload_data_product_version      = lookup(var.reload_data_product_versions, local.environment)
-  get_schema_version               = lookup(var.get_schema_versions, local.environment)
-  create_schema_version            = lookup(var.create_schema_versions, local.environment)
-  landing_to_raw_version           = lookup(var.landing_to_raw_versions, local.environment)
-  update_metadata_version          = lookup(var.update_metadata_versions, local.environment)
-  update_schema_version            = lookup(var.update_schema_versions, local.environment)
-  preview_data_version             = lookup(var.preview_data_versions, local.environment)
+  docs_version                          = lookup(var.docs_versions, local.environment)
+  authorizer_version                    = lookup(var.authorizer_versions, local.environment)
+  get_glue_metadata_version             = lookup(var.get_glue_metadata_versions, local.environment)
+  presigned_url_version                 = lookup(var.presigned_url_versions, local.environment)
+  athena_load_version                   = lookup(var.athena_load_versions, local.environment)
+  create_metadata_version               = lookup(var.create_metadata_versions, local.environment)
+  resync_unprocessed_files_version      = lookup(var.resync_unprocessed_files_versions, local.environment)
+  reload_data_product_version           = lookup(var.reload_data_product_versions, local.environment)
+  get_schema_version                    = lookup(var.get_schema_versions, local.environment)
+  create_schema_version                 = lookup(var.create_schema_versions, local.environment)
+  landing_to_raw_version                = lookup(var.landing_to_raw_versions, local.environment)
+  update_metadata_version               = lookup(var.update_metadata_versions, local.environment)
+  update_schema_version                 = lookup(var.update_schema_versions, local.environment)
+  preview_data_version                  = lookup(var.preview_data_versions, local.environment)
+  delete_table_for_data_product_version = lookup(var.delete_table_for_data_product_versions, local.environment)
 
   # Environment vars that are used by many lambdas
   logger_environment_vars = {

diff --git a/terraform/environments/data-platform/tests/delete_table.py b/terraform/environments/data-platform/tests/delete_table.py
@@ -0,0 +1,37 @@
+import json
+import os
+import sys
+
+import boto3
+import requests
+
+
+data_product_name = "example_prison_data_product"
+table_name = "testing"
+base_url = "https://hsolkci589.execute-api.eu-west-2.amazonaws.com/development"
+schema_url = f"/data-product/{data_product_name}/table/{table_name}"
+url = base_url + schema_url
+glue = boto3.client("glue")
+
+try:
+    auth_token = json.loads(os.environ["API_AUTH"])
+    auth_token = auth_token["auth-token"]
+except KeyError:
+    print("API_AUTH environment variable should be set to a json containing auth-token")
+    sys.exit(1)
+
+headers = {"authorizationToken": auth_token}
+
+# Delete schema request
+response = requests.delete(
+    url=url,
+    headers=headers,
+)
+
+if response.status_code != 200:
+    print(f"Error deleting data product schema. Status code: {response.status_code}")
+    print(f"Error deleting data product schema. Response: {response.text}")
+    print("Exiting...")
+    sys.exit(1)
+
+response_json = response.json()
diff --git a/terraform/environments/data-platform/tests/run_smoke_tests.py b/terraform/environments/data-platform/tests/run_smoke_tests.py
@@ -17,11 +17,10 @@
 glue = boto3.client("glue")
 
 try:
-    auth_token = json.loads(os.environ['API_AUTH'])
-    print(auth_token.keys())
-    auth_token = auth_token['auth-token']
+    auth_token = json.loads(os.environ["API_AUTH"])
+    auth_token = auth_token["auth-token"]
 except KeyError:
-    print('API_AUTH environment variable should be set to a json containing auth-token')
+    print("API_AUTH environment variable should be set to a json containing auth-token")
     sys.exit(1)
 
 

diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf
@@ -54,3 +54,6 @@ variable "preview_data_versions" {
   type = map(any)
 }
 
+variable "delete_table_for_data_product_versions" {
+  type = map(any)
+}