From 81cc0209df9ccaf5c2b7bd57d8aa5fc88734b69b Mon Sep 17 00:00:00 2001 From: Josh Liburdi Date: Mon, 18 Dec 2023 04:48:49 +0000 Subject: [PATCH] build(terraform): Add CloudWatch Modules --- build/terraform/aws/README.md | 10 ++ .../aws/cloudwatch/destination/_variables.tf | 22 +++ .../aws/cloudwatch/destination/_versions.tf | 10 ++ .../aws/cloudwatch/destination/main.tf | 129 ++++++++++++++++++ .../aws/cloudwatch/destination/output.tf | 7 + .../aws/cloudwatch/subscription/_variables.tf | 16 +++ .../aws/cloudwatch/subscription/_versions.tf | 10 ++ .../aws/cloudwatch/subscription/main.tf | 12 ++ examples/build/terraform/aws/README.md | 91 ++++++++++-- .../cross_account_cross_region/build.sh | 59 ++++++++ .../config/consumer/config.jsonnet | 11 ++ .../cross_account_cross_region/destroy.sh | 10 ++ .../terraform/_provider.tf | 10 ++ .../terraform/_resources.tf | 129 ++++++++++++++++++ .../terraform/autoscaler.tf | 42 ++++++ .../terraform/consumer.tf | 33 +++++ .../aws/cloudwatch_logs/to_lambda/build.sh | 55 ++++++++ .../to_lambda/config/consumer/config.jsonnet | 15 ++ .../aws/cloudwatch_logs/to_lambda/destroy.sh | 10 ++ .../to_lambda/terraform/_provider.tf | 4 + .../to_lambda/terraform/_resources.tf | 44 ++++++ .../to_lambda/terraform/consumer.tf | 51 +++++++ 22 files changed, 766 insertions(+), 14 deletions(-) create mode 100644 build/terraform/aws/cloudwatch/destination/_variables.tf create mode 100644 build/terraform/aws/cloudwatch/destination/_versions.tf create mode 100644 build/terraform/aws/cloudwatch/destination/main.tf create mode 100644 build/terraform/aws/cloudwatch/destination/output.tf create mode 100644 build/terraform/aws/cloudwatch/subscription/_variables.tf create mode 100644 build/terraform/aws/cloudwatch/subscription/_versions.tf create mode 100644 build/terraform/aws/cloudwatch/subscription/main.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/build.sh create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/config/consumer/config.jsonnet create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/destroy.sh create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/to_lambda/build.sh create mode 100644 examples/build/terraform/aws/cloudwatch_logs/to_lambda/config/consumer/config.jsonnet create mode 100644 examples/build/terraform/aws/cloudwatch_logs/to_lambda/destroy.sh create mode 100644 examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_resources.tf create mode 100644 examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf diff --git a/build/terraform/aws/README.md b/build/terraform/aws/README.md index 5922e29d..b0d6448a 100644 --- a/build/terraform/aws/README.md +++ b/build/terraform/aws/README.md @@ -26,6 +26,16 @@ This module creates an API Gateway that sends a record to a Kinesis Data Stream. This module creates an API Gateway that invokes and sends a record to a Lambda function. +### CloudWatch + +#### Destination + +This module creates a CloudWatch Logs destination that can be used to receive logs from any AWS account or region and send them to a destination. + +#### Subscription + +This module creates a CloudWatch Logs subscription filter that can be used to send logs from a CloudWatch Logs group to a destination. Use this with the `Destination` module to send logs from any AWS account or region to a single destination. + ### DynamoDB This module is used as a template for deploying new DynamoDB tables with autoscaling enabled. These tables have [time to live](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/TTL.html) enabled and users can optionally use it by writing values to the `ttl` column. diff --git a/build/terraform/aws/cloudwatch/destination/_variables.tf b/build/terraform/aws/cloudwatch/destination/_variables.tf new file mode 100644 index 00000000..b19fd863 --- /dev/null +++ b/build/terraform/aws/cloudwatch/destination/_variables.tf @@ -0,0 +1,22 @@ +variable "kms" { + type = object({ + arn = string + id = string + }) + description = "KMS key used to encrypt the resources." +} + +variable "config" { + type = object({ + name = string + destination_arn = string + account_ids = optional(list(string), []) + }) + + description = "Configuration for the CloudWatch destination." +} + +variable "tags" { + type = map(any) + default = {} +} diff --git a/build/terraform/aws/cloudwatch/destination/_versions.tf b/build/terraform/aws/cloudwatch/destination/_versions.tf new file mode 100644 index 00000000..59c42e8d --- /dev/null +++ b/build/terraform/aws/cloudwatch/destination/_versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} diff --git a/build/terraform/aws/cloudwatch/destination/main.tf b/build/terraform/aws/cloudwatch/destination/main.tf new file mode 100644 index 00000000..3feb1c47 --- /dev/null +++ b/build/terraform/aws/cloudwatch/destination/main.tf @@ -0,0 +1,129 @@ +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} + +locals { + # By default, the current account is included in the list of accounts. + account_ids = concat(var.config.account_ids, [data.aws_caller_identity.current.account_id]) +} + +data "aws_iam_policy_document" "destination_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = [ + "logs.amazonaws.com" + ] + } + + condition { + test = "StringLike" + variable = "aws:SourceArn" + + # Creates a list of wildcarded ARNs for each account. + values = formatlist("arn:aws:logs:*:%s:*", local.account_ids) + } + } +} + +data "aws_iam_policy_document" "destination" { + statement { + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey" + ] + + // Access the KMS key. + resources = [ + var.kms.arn, + ] + } + + // If the destination is Kinesis Firehose, the role must have write access. + dynamic "statement" { + for_each = strcontains(var.config.destination_arn, "arn:aws:firehose:") ? [1] : [] + + content { + effect = "Allow" + actions = [ + "firehose:DescribeDeliveryStream", + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + + resources = [ + var.config.destination_arn, + ] + } + } + + // If the destination is Kinesis Data Stream, the role must have write access. + dynamic "statement" { + for_each = strcontains(var.config.destination_arn, "arn:aws:kinesis:") ? [1] : [] + + content { + effect = "Allow" + actions = [ + "kinesis:DescribeStream", + "kinesis:DescribeStreamSummary", + "kinesis:DescribeStreamConsumer", + "kinesis:SubscribeToShard", + "kinesis:RegisterStreamConsumer", + "kinesis:PutRecord", + "kinesis:PutRecords", + ] + + resources = [ + var.config.destination_arn, + ] + } + } +} + +resource "aws_iam_role" "destination" { + name = "sub-cloudwatch-destination-${var.config.name}-${data.aws_region.current.name}" + assume_role_policy = data.aws_iam_policy_document.destination_assume_role.json + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "destination" { + role = aws_iam_role.destination.name + policy_arn = aws_iam_policy.destination.arn +} + +resource "aws_iam_policy" "destination" { + name = "sub-cloudwatch-destination-${var.config.name}-${data.aws_region.current.name}" + description = "Policy for the ${var.config.name} CloudWatch destination." + policy = data.aws_iam_policy_document.destination.json +} + +resource "aws_cloudwatch_log_destination" "destination" { + name = var.config.name + role_arn = aws_iam_role.destination.arn + target_arn = var.config.destination_arn +} + +data "aws_iam_policy_document" "destination_access" { + statement { + effect = "Allow" + principals { + type = "AWS" + identifiers = local.account_ids + } + + actions = [ + "logs:PutSubscriptionFilter", + ] + resources = [ + aws_cloudwatch_log_destination.destination.arn, + ] + } +} + +resource "aws_cloudwatch_log_destination_policy" "destination" { + destination_name = aws_cloudwatch_log_destination.destination.name + access_policy = data.aws_iam_policy_document.destination_access.json +} diff --git a/build/terraform/aws/cloudwatch/destination/output.tf b/build/terraform/aws/cloudwatch/destination/output.tf new file mode 100644 index 00000000..247a035b --- /dev/null +++ b/build/terraform/aws/cloudwatch/destination/output.tf @@ -0,0 +1,7 @@ +output "role" { + value = aws_iam_role.destination +} + +output "arn" { + value = aws_cloudwatch_log_destination.destination.arn +} diff --git a/build/terraform/aws/cloudwatch/subscription/_variables.tf b/build/terraform/aws/cloudwatch/subscription/_variables.tf new file mode 100644 index 00000000..f567daf6 --- /dev/null +++ b/build/terraform/aws/cloudwatch/subscription/_variables.tf @@ -0,0 +1,16 @@ +variable "config" { + type = object({ + name = string + destination_arn = string + log_groups = list(string) + filter_pattern = optional(string, "") + + }) + + description = "Configuration for the CloudWatch subscription filter." +} + +variable "tags" { + type = map(any) + default = {} +} diff --git a/build/terraform/aws/cloudwatch/subscription/_versions.tf b/build/terraform/aws/cloudwatch/subscription/_versions.tf new file mode 100644 index 00000000..fe6a7deb --- /dev/null +++ b/build/terraform/aws/cloudwatch/subscription/_versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.2" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} diff --git a/build/terraform/aws/cloudwatch/subscription/main.tf b/build/terraform/aws/cloudwatch/subscription/main.tf new file mode 100644 index 00000000..582ddb71 --- /dev/null +++ b/build/terraform/aws/cloudwatch/subscription/main.tf @@ -0,0 +1,12 @@ +resource "aws_cloudwatch_log_subscription_filter" "subscription_filter" { + for_each = toset(var.config.log_groups) + log_group_name = each.key + + name = var.config.name + destination_arn = var.config.destination_arn + # By default there is no filter pattern, so all logs are sent to the destination. + filter_pattern = var.config.filter_pattern + + # If the destination is a Kinesis stream, then randomly distribute the logs to avoid hot shards. + distribution = "Random" +} diff --git a/examples/build/terraform/aws/README.md b/examples/build/terraform/aws/README.md index c11f5668..019fbf56 100644 --- a/examples/build/terraform/aws/README.md +++ b/examples/build/terraform/aws/README.md @@ -2,6 +2,64 @@ These example deployments demonstrate different use cases for Substation on AWS. +# CloudWatch Logs + +## Cross-Account / Cross-Region + +Deploys a data pipeline that collects data from CloudWatch log groups in any account or region into a Kinesis Data Stream. + +```mermaid + +flowchart LR + %% resources + cw1([CloudWatch Log Group]) + cw2([CloudWatch Log Group]) + cw3([CloudWatch Log Group]) + kds([Kinesis Data Stream]) + + consumerHandler[[Handler]] + consumerTransforms[Transforms] + + subgraph Account B / Region us-west-2 + cw2 + end + + subgraph Account A / Region us-west-2 + cw3 + end + + subgraph Account A / Region us-east-1 + cw1 --> kds + cw3 --> kds + cw2 --> kds + kds --> consumerHandler + + subgraph Substation Consumer Node + consumerHandler --> consumerTransforms + end + end +``` + +## To Lambda + +Deploys a data pipeline that sends data from a CloudWatch log group to a Lambda function. + +```mermaid + +flowchart LR + %% resources + cw([CloudWatch Log Group]) + + consumerHandler[[Handler]] + consumerTransforms[Transforms] + + cw --> consumerHandler + + subgraph Substation Consumer Node + consumerHandler --> consumerTransforms + end +``` + # DynamoDB ## Change Data Capture (CDC) @@ -60,26 +118,31 @@ Deploys a data pipeline that implements a multi-phase streaming data pattern usi flowchart LR %% resources - gateway([API Gateway]) - kds1([Kinesis Data Stream]) - kds2([Kinesis Data Stream]) + cw1([CloudWatch Log Group]) + cw2([CloudWatch Log Group]) + cw3([CloudWatch Log Group]) + kds([Kinesis Data Stream]) - publisherHandler[[Handler]] - publisherTransforms[Transforms] + consumerHandler[[Handler]] + consumerTransforms[Transforms] - subscriberHandler[[Handler]] - subscriberTransforms[Transforms] + subgraph Account B / Region us-west-2 + cw2 + end - %% connections - gateway --> kds1 --> publisherHandler - subgraph Substation Publisher Node - publisherHandler --> publisherTransforms + subgraph Account A / Region us-west-2 + cw3 end - publisherTransforms --> kds2 --> subscriberHandler + subgraph Account A / Region us-east-1 + cw1 --> kds + cw3 --> kds + cw2 --> kds + kds --> consumerHandler - subgraph Substation Subscriber Node - subscriberHandler --> subscriberTransforms + subgraph Substation Consumer Node + consumerHandler --> consumerTransforms + end end ``` diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/build.sh b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/build.sh new file mode 100644 index 00000000..f8bdce7f --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/build.sh @@ -0,0 +1,59 @@ +if [ -z "$SUBSTATION_ROOT" ] +then + >&2 echo "Error: SUBSTATION_ROOT not set. This is the root directory of the Substation repository." + exit 1 +fi + +if [ -z "$AWS_ACCOUNT_ID" ] +then + >&2 echo "Error: AWS_ACCOUNT_ID not set." + exit 1 +fi + +if [ -z "$AWS_REGION" ] +then + >&2 echo "Error: AWS_REGION not set." + exit 1 +fi + +if [ -z "$AWS_ARCHITECTURE" ] +then + >&2 echo "Error: AWS_ARCHITECTURE is empty and must be set. Valid values are x86_64 and arm64." + exit 1 +fi + +export AWS_DEFAULT_REGION=$AWS_REGION +BUILD_DIR=$SUBSTATION_ROOT/examples/build/terraform/aws/cloudwatch/cross_account_cross_region + +echo "> Deploying infrastructure in AWS with Terraform" && \ +cd $BUILD_DIR/terraform && \ +terraform init && \ +terraform apply \ +-target=module.kms_substation \ +-target=aws_appconfig_application.substation \ +-target=aws_appconfig_environment.prod \ +-target=aws_appconfig_environment.dev \ +-target=aws_appconfig_deployment_strategy.instant \ +-target=module.ecr_substation \ +-target=module.ecr_autoscaling \ + +echo "> Building Substation container images and pushing to AWS ECR" && \ +cd $SUBSTATION_ROOT && \ +sh build/scripts/aws/lambda/get_appconfig_extension.sh && \ +docker build --build-arg AWS_ARCHITECTURE=$AWS_ARCHITECTURE -f build/container/aws/lambda/substation/Dockerfile -t substation:latest-$AWS_ARCHITECTURE . && \ +docker build --build-arg AWS_ARCHITECTURE=$AWS_ARCHITECTURE -f build/container/aws/lambda/autoscaling/Dockerfile -t autoscaler:latest-$AWS_ARCHITECTURE . && \ +# In production environments the tag should match the version of Substation being deployed +docker tag substation:latest-$AWS_ARCHITECTURE $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/substation:latest && \ +docker tag autoscaler:latest-$AWS_ARCHITECTURE $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/autoscaler:latest && \ +sh build/scripts/aws/ecr_login.sh && \ +docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/substation:latest && \ +docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/autoscaler:latest + +echo "> Deploying Substation nodes with Terraform" && \ +cd $BUILD_DIR/terraform && \ +terraform apply + +echo "> Compiling Substation configurations and uploading to AWS AppConfig" && \ +cd $SUBSTATION_ROOT && \ +sh build/scripts/config/compile.sh && \ +SUBSTATION_CONFIG_DIRECTORY=$BUILD_DIR AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_ENVIRONMENT=example AWS_APPCONFIG_DEPLOYMENT_STRATEGY=Instant python3 build/scripts/aws/appconfig/appconfig_upload.py diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/config/consumer/config.jsonnet b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/config/consumer/config.jsonnet new file mode 100644 index 00000000..a58f2bbb --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/config/consumer/config.jsonnet @@ -0,0 +1,11 @@ +local sub = import '../../../../../../../../build/config/substation.libsonnet'; + +{ + concurrency: 1, + transforms: [ + // CloudWatch logs sent to Kinesis Data Streams are gzip compressed. + // These must be decompressed before other transforms are applied. + sub.tf.fmt.from.gzip(), + sub.tf.send.stdout(), + ], +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/destroy.sh b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/destroy.sh new file mode 100644 index 00000000..2492d1c8 --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/destroy.sh @@ -0,0 +1,10 @@ +export AWS_DEFAULT_REGION=$AWS_REGION +BUILD_DIR=$SUBSTATION_ROOT/examples/build/terraform/aws/cloudwatch/cross_account_cross_region + +echo "> Removing Substation configurations from AWS AppConfig" && \ +cd $SUBSTATION_ROOT && \ +AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_PROFILE_NAME=consumer python3 build/scripts/aws/appconfig/appconfig_delete.py + +echo "> Destroying infrastructure in AWS with Terraform" && \ +cd $BUILD_DIR/terraform && \ +terraform destroy diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf new file mode 100644 index 00000000..af2358d2 --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_provider.tf @@ -0,0 +1,10 @@ +provider "aws" { + # profile = "default" + region = "us-east-1" +} + +provider "aws" { + # profile = "default" + alias = "usw2" + region = "us-west-2" +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf new file mode 100644 index 00000000..8474210e --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/_resources.tf @@ -0,0 +1,129 @@ +data "aws_caller_identity" "caller" {} + +# KMS encryption key that is shared by all Substation infrastructure +module "kms" { + source = "../../../../../../../build/terraform/aws/kms" + + config = { + name = "alias/substation" + } +} + +# AppConfig application that is shared by all Substation applications. +resource "aws_appconfig_application" "substation" { + name = "substation" + description = "Stores compiled configuration files for Substation" +} + +resource "aws_appconfig_environment" "example" { + name = "example" + description = "Stores example Substation configuration files" + application_id = aws_appconfig_application.substation.id +} + +# AWS Lambda requires an instant deployment strategy. +resource "aws_appconfig_deployment_strategy" "instant" { + name = "Instant" + description = "This strategy deploys the configuration to all targets immediately with zero bake time." + deployment_duration_in_minutes = 0 + final_bake_time_in_minutes = 0 + growth_factor = 100 + growth_type = "LINEAR" + replicate_to = "NONE" +} + +# Repository for the core Substation application. +module "ecr_substation" { + source = "../../../../../../../build/terraform/aws/ecr" + kms = module.kms + + config = { + name = "substation" + force_delete = true + } +} + +# Repository for the autoscaling application. +module "ecr_autoscaling" { + source = "../../../../../../../build/terraform/aws/ecr" + kms = module.kms + + config = { + name = "autoscaler" + force_delete = true + } +} + +# SNS topic for Kinesis Data Stream autoscaling alarms. +resource "aws_sns_topic" "autoscaling_topic" { + name = "autoscaler" + kms_master_key_id = module.kms.id +} + +# Kinesis Data Stream that is used as the destination for CloudWatch Logs. +module "kds" { + source = "../../../../../../../build/terraform/aws/kinesis_data_stream" + kms = module.kms + + config = { + name = "substation" + autoscaling_topic = aws_sns_topic.autoscaling_topic.arn + } + + access = [ + # Autoscales the stream. + module.lambda_autoscaling.role.name, + # Reads data from the stream. + module.lambda_consumer.role.name, + # Writes data to the stream. + module.cw_destination_use1.role.name, + module.cw_destination_usw2.role.name, + ] +} + +# CloudWatch Logs destination that sends logs to the Kinesis Data Stream from us-east-1. +module "cw_destination_use1" { + source = "../../../../../../../build/terraform/aws/cloudwatch/destination" + + kms = module.kms + config = { + name = "substation" + destination_arn = module.kds.arn + + # By default, any CloudWatch log in the current AWS account can send logs to this destination. + # Add additional AWS account IDs to allow them to send logs to the destination. + account_ids = [] + } +} + +module "cw_subscription_use1" { + source = "../../../../../../../build/terraform/aws/cloudwatch/subscription" + + config = { + name = "substation" + destination_arn = module.cw_destination_use1.arn + log_groups = [ + # This example causes recursion. Add other log groups for resources in us-east-1. + # "/aws/lambda/consumer", + ] + } +} + +# CloudWatch Logs destination that sends logs to the Kinesis Data Stream from us-west-2. +# To add support for more regions, copy this module and change the provider. +module "cw_destination_usw2" { + source = "../../../../../../../build/terraform/aws/cloudwatch/destination" + providers = { + aws = aws.usw2 + } + + kms = module.kms + config = { + name = "substation" + destination_arn = module.kds.arn + + # By default, any CloudWatch log in the current AWS account can send logs to this destination. + # Add additional AWS account IDs to allow them to send logs to the destination. + account_ids = [] + } +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf new file mode 100644 index 00000000..d02326c4 --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/autoscaler.tf @@ -0,0 +1,42 @@ +# Used for deploying and maintaining the Kinesis Data Streams autoscaling application; does not need to be used if deployments don't include Kinesis Data Streams. + +module "lambda_autoscaling" { + source = "../../../../../../../build/terraform/aws/lambda" + # These are always required for all Lambda. + kms = module.kms + appconfig = aws_appconfig_application.substation + + config = { + name = "autoscaler" + description = "Autoscaler for Kinesis Data Streams" + image_uri = "${module.ecr_autoscaling.url}:latest" + image_arm = true + } + + depends_on = [ + aws_appconfig_application.substation, + module.ecr_autoscaling.url, + ] +} + +resource "aws_sns_topic_subscription" "autoscaling_subscription" { + topic_arn = aws_sns_topic.autoscaling_topic.arn + protocol = "lambda" + endpoint = module.lambda_autoscaling.arn + + depends_on = [ + module.lambda_autoscaling.name + ] +} + +resource "aws_lambda_permission" "autoscaling_invoke" { + statement_id = "AllowExecutionFromSNS" + action = "lambda:InvokeFunction" + function_name = module.lambda_autoscaling.name + principal = "sns.amazonaws.com" + source_arn = aws_sns_topic.autoscaling_topic.arn + + depends_on = [ + module.lambda_autoscaling.name + ] +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf new file mode 100644 index 00000000..683c283c --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/cross_account_cross_region/terraform/consumer.tf @@ -0,0 +1,33 @@ +module "lambda_consumer" { + source = "../../../../../../../build/terraform/aws/lambda" + # These are always required for all Lambda. + kms = module.kms + appconfig = aws_appconfig_application.substation + + config = { + name = "consumer" + description = "Substation node that consumes from Kinesis" + image_uri = "${module.ecr_substation.url}:latest" + image_arm = true + + env = { + "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/consumer" + "SUBSTATION_HANDLER" : "AWS_KINESIS_DATA_STREAM" + "SUBSTATION_DEBUG" : true + } + } + + depends_on = [ + aws_appconfig_application.substation, + module.ecr_substation.url, + ] +} + +resource "aws_lambda_event_source_mapping" "lambda_consumer" { + event_source_arn = module.kds.arn + function_name = module.lambda_consumer.arn + maximum_batching_window_in_seconds = 10 + batch_size = 100 + parallelization_factor = 1 + starting_position = "LATEST" +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/to_lambda/build.sh b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/build.sh new file mode 100644 index 00000000..d7df1f78 --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/build.sh @@ -0,0 +1,55 @@ +if [ -z "$SUBSTATION_ROOT" ] +then + >&2 echo "Error: SUBSTATION_ROOT not set. This is the root directory of the Substation repository." + exit 1 +fi + +if [ -z "$AWS_ACCOUNT_ID" ] +then + >&2 echo "Error: AWS_ACCOUNT_ID not set." + exit 1 +fi + +if [ -z "$AWS_REGION" ] +then + >&2 echo "Error: AWS_REGION not set." + exit 1 +fi + +if [ -z "$AWS_ARCHITECTURE" ] +then + >&2 echo "Error: AWS_ARCHITECTURE is empty and must be set. Valid values are x86_64 and arm64." + exit 1 +fi + +export AWS_DEFAULT_REGION=$AWS_REGION +BUILD_DIR=$SUBSTATION_ROOT/examples/build/terraform/aws/cloudwatch/to_lambda + +echo "> Deploying infrastructure in AWS with Terraform" && \ +cd $BUILD_DIR/terraform && \ +terraform init && \ +terraform apply \ +-target=module.kms_substation \ +-target=aws_appconfig_application.substation \ +-target=aws_appconfig_environment.prod \ +-target=aws_appconfig_environment.dev \ +-target=aws_appconfig_deployment_strategy.instant \ +-target=module.ecr_substation \ + +echo "> Building Substation container images and pushing to AWS ECR" && \ +cd $SUBSTATION_ROOT && \ +sh build/scripts/aws/lambda/get_appconfig_extension.sh && \ +docker build --build-arg AWS_ARCHITECTURE=$AWS_ARCHITECTURE -f build/container/aws/lambda/substation/Dockerfile -t substation:latest-$AWS_ARCHITECTURE . && \ +# In production environments the tag should match the version of Substation being deployed +docker tag substation:latest-$AWS_ARCHITECTURE $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/substation:latest && \ +sh build/scripts/aws/ecr_login.sh && \ +docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/substation:latest + +echo "> Deploying Substation nodes with Terraform" && \ +cd $BUILD_DIR/terraform && \ +terraform apply + +echo "> Compiling Substation configurations and uploading to AWS AppConfig" && \ +cd $SUBSTATION_ROOT && \ +sh build/scripts/config/compile.sh && \ +SUBSTATION_CONFIG_DIRECTORY=$BUILD_DIR AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_ENVIRONMENT=example AWS_APPCONFIG_DEPLOYMENT_STRATEGY=Instant python3 build/scripts/aws/appconfig/appconfig_upload.py diff --git a/examples/build/terraform/aws/cloudwatch_logs/to_lambda/config/consumer/config.jsonnet b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/config/consumer/config.jsonnet new file mode 100644 index 00000000..b2b56948 --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/config/consumer/config.jsonnet @@ -0,0 +1,15 @@ +local sub = import '../../../../../../../../build/config/substation.libsonnet'; + +{ + concurrency: 1, + transforms: [ + // CloudWatch logs sent to Lambda are base64 encoded and gzip + // compressed within the `awslogs.data` field of the event. + // These must be decoded and decompressed before other transforms are + // applied. + sub.tf.obj.cp({obj: {key: 'awslogs.data'}}), + sub.tf.fmt.from.base64(), + sub.tf.fmt.from.gzip(), + sub.tf.send.stdout(), + ], +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/to_lambda/destroy.sh b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/destroy.sh new file mode 100644 index 00000000..eb87029e --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/destroy.sh @@ -0,0 +1,10 @@ +export AWS_DEFAULT_REGION=$AWS_REGION +BUILD_DIR=$SUBSTATION_ROOT/examples/build/terraform/aws/cloudwatch/to_lambda + +echo "> Removing Substation configurations from AWS AppConfig" && \ +cd $SUBSTATION_ROOT && \ +AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_PROFILE_NAME=consumer python3 build/scripts/aws/appconfig/appconfig_delete.py + +echo "> Destroying infrastructure in AWS with Terraform" && \ +cd $BUILD_DIR/terraform && \ +terraform destroy diff --git a/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf new file mode 100644 index 00000000..872a6309 --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + # profile = "default" + region = "us-east-1" +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_resources.tf b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_resources.tf new file mode 100644 index 00000000..658eb72b --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/_resources.tf @@ -0,0 +1,44 @@ +data "aws_caller_identity" "caller" {} + +# KMS encryption key that is shared by all Substation infrastructure +module "kms" { + source = "../../../../../../../build/terraform/aws/kms" + + config = { + name = "alias/substation" + } +} + +# AppConfig application that is shared by all Substation applications. +resource "aws_appconfig_application" "substation" { + name = "substation" + description = "Stores compiled configuration files for Substation" +} + +resource "aws_appconfig_environment" "example" { + name = "example" + description = "Stores example Substation configuration files" + application_id = aws_appconfig_application.substation.id +} + +# AWS Lambda requires an instant deployment strategy. +resource "aws_appconfig_deployment_strategy" "instant" { + name = "Instant" + description = "This strategy deploys the configuration to all targets immediately with zero bake time." + deployment_duration_in_minutes = 0 + final_bake_time_in_minutes = 0 + growth_factor = 100 + growth_type = "LINEAR" + replicate_to = "NONE" +} + +# Repository for the core Substation application. +module "ecr_substation" { + source = "../../../../../../../build/terraform/aws/ecr" + kms = module.kms + + config = { + name = "substation" + force_delete = true + } +} diff --git a/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf new file mode 100644 index 00000000..00c8afcd --- /dev/null +++ b/examples/build/terraform/aws/cloudwatch_logs/to_lambda/terraform/consumer.tf @@ -0,0 +1,51 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +module "lambda_consumer" { + source = "../../../../../../../build/terraform/aws/lambda" + # These are always required for all Lambda. + kms = module.kms + appconfig = aws_appconfig_application.substation + + config = { + name = "consumer" + description = "Substation node that is invoked by CloudWatch" + image_uri = "${module.ecr_substation.url}:latest" + image_arm = true + + env = { + "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/consumer" + "SUBSTATION_HANDLER" : "AWS_LAMBDA" + "SUBSTATION_DEBUG" : true + } + } + + depends_on = [ + aws_appconfig_application.substation, + module.ecr_substation.url, + ] +} + +# Allows any CloudWatch log group to send logs to the Lambda function in the current AWS account and region. +# Repeat this for each region that sends logs to the Lambda function. +resource "aws_lambda_permission" "consumer" { + statement_id = "AllowExecutionFromCloudWatch" + action = "lambda:InvokeFunction" + function_name = module.lambda_consumer.name + principal = "logs.amazonaws.com" + source_arn = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:*" +} + +# CloudWatch Logs subscription filter that sends logs to the Lambda function. +module "cw_subscription" { + source = "../../../../../../../build/terraform/aws/cloudwatch/subscription" + + config = { + name = "substation" + destination_arn = module.lambda_consumer.arn + log_groups = [ + # This example causes recursion. Add other log groups for resources in the account and region. + "/aws/lambda/test", + ] + } +}