diff --git a/README.md b/README.md index 0a09f4a..81df256 100755 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ |-|-| ### **Description**: -This repo contains the CDK Project that sets up all of the infrastructure the file processing pipeline requires. +This repo contains the CDK Project that sets up all of the infrastructure the file processing pipeline requires. This includes S3 Buckets, Public and Private ECR Repos, Timestream databases, Postgres RDS Servers, as well as Lambda Functions. # Information on working with a CDK Project diff --git a/buildspec.yml b/buildspec.yml index aa547df..981bf0f 100755 --- a/buildspec.yml +++ b/buildspec.yml @@ -3,46 +3,59 @@ version: 0.2 phases: pre_build: commands: - - echo Install dev dependencies + # Install dependencies + - echo "Installing dev dependencies" + - apt-get update + - apt-get install -y python3-pip make graphviz unzip - pip3 install -r requirements.txt - - npm install -g aws-cdk@2.77.0 - - Docs test - - apt-get update && apt-get install -y python3-pip - - apt-get install make graphviz -y + # Documentation Test + - echo "Generating documentation" - make html - - - echo Lint with Black - - black --check --diff app.py cdk_deployment - - echo Lint with Flake - - flake8 --count --max-line-length 88 app.py cdk_deployment - - - echo CDK Test - - cdk synth + # Install Terraform + - echo "Installing Terraform" + - cd terraform + - curl -o terraform.zip https://releases.hashicorp.com/terraform/1.4.6/terraform_1.4.6_linux_amd64.zip + - unzip -o terraform.zip + - sudo mv terraform /usr/local/bin/ + - terraform --version + - terraform init && terraform validate - - build: commands: - - echo Deploying Bootstrap Architecture... - - cdk bootstrap -y --require-approval never + # Deployment commands + - echo "Deploying Bootstrap Architecture..." - | - if [[ -z "${LAMBDA_PIPELINE}" ]];then - echo Deploying Pipeline Architecture... - cdk deploy SDCAWSPipelineArchitectureStack -y --require-approval never - echo Start other Codebuilds - aws codebuild start-build --project-name build_sdc_aws_base_docker_image --region us-east-2 --environment-variables-override name=DEPLOYMENT_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT - aws codebuild start-build --project-name build_sdc_aws_sorting_lambda --region us-east-2 --environment-variables-override name=DEPLOYMENT_ENVIRONMENT,value=$CDK_ENVIRONMENT,type=PLAINTEXT + if git describe --tags --exact-match > /dev/null 2>&1; then + echo "This is a tag push event" + PF_ECR_REPO=sdc_aws_processing_lambda + CDK_ENVIRONMENT=PRODUCTION + SF_ECR_REPO=sdc_aws_sorting_lambda + terraform workspace select prod + elif [ "${CDK_ENVIRONMENT}" = "PRODUCTION" ]; then + echo "This is a production environment" + PF_ECR_REPO=sdc_aws_processing_lambda + SF_ECR_REPO=sdc_aws_sorting_lambda + terraform workspace select prod else - if [[ $LAMBDA_PIPELINE == *"SORTING"* ]];then - echo Deploying Sorting Lambda - cdk deploy SDCAWSSortingLambdaStack -y --require-approval never - else - echo Deploying Processing Lambda - cdk deploy SDCAWSProcessingLambdaStack -y --require-approval never - fi - fi - - - - echo Build completed on `date` \ No newline at end of file + echo "This is a development environment" + PF_ECR_REPO=dev-sdc_aws_processing_lambda + SF_ECR_REPO=dev-sdc_aws_sorting_lambda + terraform workspace select dev + fi + + # Fetch latest image and SF image tag + - | + PF_IMAGE_TAG=$(aws ecr describe-images --repository-name $PF_ECR_REPO --region us-east-1 --query "sort_by(imageDetails,& imagePushedAt)[-1].imageTags[]" --output text | awk '{for(i=1;i<=NF;i++) if($i!="latest") print $i; exit}') + echo $PF_IMAGE_TAG + + SF_IMAGE_TAG=$(aws ecr describe-images --repository-name $SF_ECR_REPO --region us-east-1 --query "sort_by(imageDetails,& imagePushedAt)[-1].imageTags[]" --output text | awk '{for(i=1;i<=NF;i++) if($i!="latest") print $i; exit}') + echo $SF_IMAGE_TAG + + + # Run Terraform apply + - terraform apply -auto-approve -var "pf_image_tag=$PF_IMAGE_TAG" -var "sf_image_tag=$SF_IMAGE_TAG" + + # Completion message + - echo "Build completed on $(date)" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 5ff622d..929e598 100755 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ flake8==6.0.0 black==23.3.0 sphinx==6.1.3 sphinx-automodapi==0.15.0 -sphinx_rtd_theme==1.2.0 \ No newline at end of file +sphinx_rtd_theme==1.2.0 +pyyaml==6.0.0 diff --git a/terraform/.terraform-version b/terraform/.terraform-version new file mode 100644 index 0000000..6463e95 --- /dev/null +++ b/terraform/.terraform-version @@ -0,0 +1 @@ +1.6.4 \ No newline at end of file diff --git a/terraform/config.auto.tfvars b/terraform/config.auto.tfvars index f59a92b..fc6ef18 100644 --- a/terraform/config.auto.tfvars +++ b/terraform/config.auto.tfvars @@ -25,9 +25,9 @@ timestream_s3_logs_table_name = "sdc_aws_s3_bucket_log_table" # The names of the buckets that will be created for the mission incoming_bucket_name = "swsoc-incoming" -# S3 Sorting Lambda Bucket Name -# The name of the bucket that will be created to store the build artifacts for the sorting lambda -sorting_lambda_bucket_name = "swsoc-sorting-lambda" +# S3 Sorting Lambda ECR Repository Name +# The name of the ECR repository that will be created to store the sorting lambda image +sorting_function_private_ecr_name = "sdc_aws_sorting_lambda" # S3 Server Access Logs Bucket # The name of the bucket that will be created to store the s3 server access logs diff --git a/terraform/main.tf b/terraform/main.tf index 4974ffa..0c06758 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -54,7 +54,7 @@ locals { last_data_level = element(var.valid_data_levels, length(var.valid_data_levels) - 1) instrument_bucket_names = [for bucket in var.instrument_names : "${var.mission_name}-${bucket}"] - bucket_list = concat([var.incoming_bucket_name], [var.sorting_lambda_bucket_name], local.instrument_bucket_names) + bucket_list = concat([var.incoming_bucket_name], local.instrument_bucket_names) } diff --git a/terraform/sdc_aws_pipeline_infrastructure.tf b/terraform/sdc_aws_pipeline_infrastructure.tf index 6ad16cd..47fb230 100644 --- a/terraform/sdc_aws_pipeline_infrastructure.tf +++ b/terraform/sdc_aws_pipeline_infrastructure.tf @@ -203,6 +203,13 @@ resource "aws_ecr_repository" "processing_function_private_ecr" { tags = local.standard_tags } +// Private ECR for the processing function +resource "aws_ecr_repository" "sorting_function_private_ecr" { + name = "${local.environment_short_name}${var.sorting_function_private_ecr_name}" + image_tag_mutability = "MUTABLE" + tags = local.standard_tags +} + // Public ECR for the docker base image resource "aws_ecrpublic_repository" "docker_base_public_ecr" { repository_name = "${local.environment_short_name}${var.docker_base_public_ecr_name}" diff --git a/terraform/sdc_aws_processing_lambda_function.tf b/terraform/sdc_aws_processing_lambda_function.tf index 0346d68..5c7d3d1 100644 --- a/terraform/sdc_aws_processing_lambda_function.tf +++ b/terraform/sdc_aws_processing_lambda_function.tf @@ -11,7 +11,7 @@ resource "aws_lambda_function" "aws_sdc_processing_lambda_function" { memory_size = 128 timeout = 900 - image_uri = "${aws_ecr_repository.processing_function_private_ecr.repository_url}:${var.image_tag}" + image_uri = "${aws_ecr_repository.processing_function_private_ecr.repository_url}:${var.pf_image_tag}" package_type = "Image" environment { diff --git a/terraform/sdc_aws_sorting_lambda_function.tf b/terraform/sdc_aws_sorting_lambda_function.tf index 548c6d0..9b1cddf 100644 --- a/terraform/sdc_aws_sorting_lambda_function.tf +++ b/terraform/sdc_aws_sorting_lambda_function.tf @@ -8,8 +8,6 @@ // Creates the Sorting Lambda function resource "aws_lambda_function" "sorting_lambda_function" { function_name = local.is_production ? "aws_sdc_sorting_lambda_function" : "dev_aws_sdc_sorting_lambda_function" - handler = "lambda_function.handler" - runtime = "python3.10" memory_size = 128 timeout = 600 @@ -21,8 +19,9 @@ resource "aws_lambda_function" "sorting_lambda_function" { } } - s3_bucket = "${local.environment_short_name}${var.sorting_lambda_bucket_name}" - s3_key = var.s3_key + image_uri = "${aws_ecr_repository.sorting_function_private_ecr.repository_url}:${var.sf_image_tag}" + package_type = "Image" + ephemeral_storage { size = 512 } @@ -31,9 +30,6 @@ resource "aws_lambda_function" "sorting_lambda_function" { mode = "PassThrough" } - architectures = ["x86_64"] - // The last object, assuming it's the latest - role = aws_iam_role.sorting_lambda_exec.arn tags = local.standard_tags diff --git a/terraform/variables.tf b/terraform/variables.tf index 0570509..bea8048 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -30,14 +30,14 @@ variable "mission_name" { description = "The list of missions" } -variable "sorting_lambda_bucket_name" { +variable "s3_server_access_logs_bucket_name" { type = string - description = "The name of the S3 bucket to create for storing the sorting lambda" + description = "The name of the S3 bucket to create for storing access logs" } -variable "s3_server_access_logs_bucket_name" { +variable "sorting_function_private_ecr_name" { type = string - description = "The name of the S3 bucket to create for storing access logs" + description = "Private ECR repository for the sorting function" } variable "processing_function_private_ecr_name" { @@ -64,15 +64,16 @@ variable "slack_channel" { sensitive = true } -variable "image_tag" { +variable "pf_image_tag" { type = string - description = "ECR image tag" + description = "Processing Function ECR image tag" default = "latest" } -variable "s3_key" { - type = string - description = "S3 key for the sorting lambda" +variable "sf_image_tag" { + type = string + description = "Sorting Function ECR image tag" + default = "latest" } variable "valid_data_levels" {