Refactor Sorting Function to a Container Function

README.md

@@ -4,7 +4,7 @@
 |-|-|
 
 ### **Description**:
-This repo contains the CDK Project that sets up all of the infrastructure the file processing pipeline requires.
+This repo contains the CDK Project that sets up all of the infrastructure the file processing pipeline requires. This includes S3 Buckets, Public and Private ECR Repos, Timestream databases, Postgres RDS Servers, as well as Lambda Functions.
 
 
 # Information on working with a CDK Project

buildspec.yml

@@ -3,46 +3,59 @@ version: 0.2
 phases:
   pre_build:
     commands:
-      - echo Install dev dependencies
+      # Install dependencies
+      - echo "Installing dev dependencies"
+      - apt-get update
+      - apt-get install -y python3-pip make graphviz unzip
       - pip3 install -r requirements.txt
-      - npm install -g [email protected]
 
-      - Docs test
-      - apt-get update && apt-get install -y python3-pip
-      - apt-get install make graphviz -y
+      # Documentation Test
+      - echo "Generating documentation"
       - make html
-
-      - echo Lint with Black
-      - black --check --diff app.py cdk_deployment
 
-      - echo Lint with Flake
-      - flake8 --count --max-line-length 88 app.py cdk_deployment
-
-      - echo CDK Test
-      - cdk synth
+      # Install Terraform
+      - echo "Installing Terraform"
+      - cd terraform
+      - curl -o terraform.zip https://releases.hashicorp.com/terraform/1.4.6/terraform_1.4.6_linux_amd64.zip
+      - unzip -o terraform.zip
+      - sudo mv terraform /usr/local/bin/
+      - terraform --version
+      - terraform init && terraform validate
 
-
-
   build:
     commands:
-      - echo Deploying Bootstrap Architecture...
-      - cdk bootstrap -y --require-approval never
+      # Deployment commands
+      - echo "Deploying Bootstrap Architecture..."
       - |
-        if [[ -z "${LAMBDA_PIPELINE}" ]];then
-          echo Deploying Pipeline Architecture...
-          cdk deploy SDCAWSPipelineArchitectureStack -y --require-approval never
-          echo Start other Codebuilds
-          aws codebuild start-build --project-name build_sdc_aws_base_docker_image --region us-east-2 --environment-variables-override name=DEPLOYMENT_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT
-          aws codebuild start-build --project-name build_sdc_aws_sorting_lambda --region us-east-2 --environment-variables-override name=DEPLOYMENT_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT
+        if git describe --tags --exact-match > /dev/null 2>&1; then
+          echo "This is a tag push event"
+          PF_ECR_REPO=sdc_aws_processing_lambda
+          CDK_ENVIRONMENT=PRODUCTION
+          SF_ECR_REPO=sdc_aws_sorting_lambda
+          terraform workspace select prod
+        elif [ "${CDK_ENVIRONMENT}" = "PRODUCTION" ]; then
+          echo "This is a production environment"
+          PF_ECR_REPO=sdc_aws_processing_lambda
+          SF_ECR_REPO=sdc_aws_sorting_lambda
+          terraform workspace select prod
         else
-          if [[ $LAMBDA_PIPELINE == *"SORTING"* ]];then
-            echo Deploying Sorting Lambda
-            cdk deploy SDCAWSSortingLambdaStack -y --require-approval never
-          else
-            echo Deploying Processing Lambda
-            cdk deploy SDCAWSProcessingLambdaStack -y --require-approval never
-          fi    
-        fi   
-
-
-      - echo Build completed on `date`
+          echo "This is a development environment"
+          PF_ECR_REPO=dev-sdc_aws_processing_lambda
+          SF_ECR_REPO=dev-sdc_aws_sorting_lambda
+          terraform workspace select dev
+        fi
+
+      # Fetch latest image and SF image tag
+      - |
+        PF_IMAGE_TAG=$(aws ecr describe-images --repository-name $PF_ECR_REPO --region us-east-1 --query "sort_by(imageDetails,& imagePushedAt)[-1].imageTags[]" --output text | awk '{for(i=1;i<=NF;i++) if($i!="latest") print $i; exit}')
+        echo $PF_IMAGE_TAG
+
+        SF_IMAGE_TAG=$(aws ecr describe-images --repository-name $SF_ECR_REPO --region us-east-1 --query "sort_by(imageDetails,& imagePushedAt)[-1].imageTags[]" --output text | awk '{for(i=1;i<=NF;i++) if($i!="latest") print $i; exit}')
+        echo $SF_IMAGE_TAG
+
+
+      # Run Terraform apply
+      - terraform apply -auto-approve -var "pf_image_tag=$PF_IMAGE_TAG" -var "sf_image_tag=$SF_IMAGE_TAG"
+
+      # Completion message
+      - echo "Build completed on $(date)"

requirements.txt

@@ -5,4 +5,5 @@ flake8==6.0.0
 black==23.3.0
 sphinx==6.1.3
 sphinx-automodapi==0.15.0
-sphinx_rtd_theme==1.2.0
+sphinx_rtd_theme==1.2.0
+pyyaml==6.0.0

terraform/.terraform-version

@@ -0,0 +1 @@
+1.6.4

terraform/config.auto.tfvars

@@ -25,9 +25,9 @@ timestream_s3_logs_table_name = "sdc_aws_s3_bucket_log_table"
 # The names of the buckets that will be created for the mission
 incoming_bucket_name = "swsoc-incoming"
 
-# S3 Sorting Lambda Bucket Name
-# The name of the bucket that will be created to store the build artifacts for the sorting lambda
-sorting_lambda_bucket_name = "swsoc-sorting-lambda"
+# S3 Sorting Lambda ECR Repository Name
+# The name of the ECR repository that will be created to store the sorting lambda image
+sorting_function_private_ecr_name = "sdc_aws_sorting_lambda"
 
 # S3 Server Access Logs Bucket
 # The name of the bucket that will be created to store the s3 server access logs

terraform/main.tf

@@ -54,7 +54,7 @@ locals {
   last_data_level = element(var.valid_data_levels, length(var.valid_data_levels) - 1)
 
   instrument_bucket_names = [for bucket in var.instrument_names : "${var.mission_name}-${bucket}"]
-  bucket_list             = concat([var.incoming_bucket_name], [var.sorting_lambda_bucket_name], local.instrument_bucket_names)
+  bucket_list             = concat([var.incoming_bucket_name], local.instrument_bucket_names)
 }
 
 

terraform/sdc_aws_pipeline_infrastructure.tf

@@ -203,6 +203,13 @@ resource "aws_ecr_repository" "processing_function_private_ecr" {
   tags                 = local.standard_tags
 }
 
+// Private ECR for the processing function
+resource "aws_ecr_repository" "sorting_function_private_ecr" {
+  name                 = "${local.environment_short_name}${var.sorting_function_private_ecr_name}"
+  image_tag_mutability = "MUTABLE"
+  tags                 = local.standard_tags
+}
+
 // Public ECR for the docker base image
 resource "aws_ecrpublic_repository" "docker_base_public_ecr" {
   repository_name = "${local.environment_short_name}${var.docker_base_public_ecr_name}"

terraform/sdc_aws_processing_lambda_function.tf

@@ -11,7 +11,7 @@ resource "aws_lambda_function" "aws_sdc_processing_lambda_function" {
   memory_size   = 128
   timeout       = 900
 
-  image_uri    = "${aws_ecr_repository.processing_function_private_ecr.repository_url}:${var.image_tag}"
+  image_uri    = "${aws_ecr_repository.processing_function_private_ecr.repository_url}:${var.pf_image_tag}"
   package_type = "Image"
 
   environment {

terraform/sdc_aws_sorting_lambda_function.tf

@@ -8,8 +8,6 @@
 // Creates the Sorting Lambda function
 resource "aws_lambda_function" "sorting_lambda_function" {
   function_name = local.is_production ? "aws_sdc_sorting_lambda_function" : "dev_aws_sdc_sorting_lambda_function"
-  handler       = "lambda_function.handler"
-  runtime       = "python3.10"
   memory_size   = 128
   timeout       = 600
 
@@ -21,8 +19,9 @@ resource "aws_lambda_function" "sorting_lambda_function" {
     }
   }
 
-  s3_bucket = "${local.environment_short_name}${var.sorting_lambda_bucket_name}"
-  s3_key    = var.s3_key
+  image_uri    = "${aws_ecr_repository.sorting_function_private_ecr.repository_url}:${var.sf_image_tag}"
+  package_type = "Image"
+
   ephemeral_storage {
     size = 512
   }
@@ -31,9 +30,6 @@ resource "aws_lambda_function" "sorting_lambda_function" {
     mode = "PassThrough"
   }
 
-  architectures = ["x86_64"]
-  // The last object, assuming it's the latest
-
   role = aws_iam_role.sorting_lambda_exec.arn
 
   tags = local.standard_tags

terraform/variables.tf

@@ -30,14 +30,14 @@ variable "mission_name" {
   description = "The list of missions"
 }
 
-variable "sorting_lambda_bucket_name" {
+variable "s3_server_access_logs_bucket_name" {
   type        = string
-  description = "The name of the S3 bucket to create for storing the sorting lambda"
+  description = "The name of the S3 bucket to create for storing access logs"
 }
 
-variable "s3_server_access_logs_bucket_name" {
+variable "sorting_function_private_ecr_name" {
   type        = string
-  description = "The name of the S3 bucket to create for storing access logs"
+  description = "Private ECR repository for the sorting function"
 }
 
 variable "processing_function_private_ecr_name" {
@@ -64,15 +64,16 @@ variable "slack_channel" {
   sensitive   = true
 }
 
-variable "image_tag" {
+variable "pf_image_tag" {
   type        = string
-  description = "ECR image tag"
+  description = "Processing Function ECR image tag"
   default     = "latest"
 }
 
-variable "s3_key" {
-  type = string
-  description = "S3 key for the sorting lambda"
+variable "sf_image_tag" {
+  type        = string
+  description = "Sorting Function ECR image tag"
+  default     = "latest"
 }
 
 variable "valid_data_levels" {