docs: S3/SNS Example

examples/build/terraform/aws/README.md

@@ -166,11 +166,32 @@ flowchart LR
     sendS3y --> bucket
 ```
 
+## SNS
+
+Deploys a data pipeline that reads data from an S3 bucket via an SNS topic.
+
+```mermaid
+
+flowchart LR
+    %% resources
+    bucket([S3 Bucket])
+    sns([SNS Topic])
+
+    handler[[Handler]]
+    transforms[Transforms]
+
+    %% connections
+    bucket --> sns --> handler
+    subgraph Substation Node 
+    handler --> transforms
+    end
+```
+
 # SNS
 
 ## Pub/Sub
 
-Deploys a data pipeline that implements a [publish/subscribe (pub/sub) pattern](https://aws.amazon.com/what-is/pub-sub-messaging/). The `cmd/client/file` Substation application can be used to send data to the SNS topic.
+Deploys a data pipeline that implements a [publish/subscribe (pub/sub) pattern](https://aws.amazon.com/what-is/pub-sub-messaging/). The `examples/cmd/client/file` application can be used to send data to the SNS topic.
 
 ```mermaid
 

examples/build/terraform/aws/s3/sns/build.sh

@@ -0,0 +1,54 @@
+if [ -z "$SUBSTATION_ROOT" ]
+then
+  >&2 echo "Error: SUBSTATION_ROOT not set. This is the root directory of the Substation repository."
+  exit 1
+fi
+
+if [ -z "$AWS_ACCOUNT_ID" ]
+then
+  >&2 echo "Error: AWS_ACCOUNT_ID not set."
+  exit 1
+fi
+
+if [ -z "$AWS_REGION" ]
+then
+  >&2 echo "Error: AWS_REGION not set."
+  exit 1
+fi
+
+if [ -z "$AWS_ARCHITECTURE" ]
+then
+  >&2 echo "Error: AWS_ARCHITECTURE is empty and must be set. Valid values are x86_64 and arm64."
+  exit 1
+fi
+
+export AWS_DEFAULT_REGION=$AWS_REGION
+BUILD_DIR=$SUBSTATION_ROOT/examples/build/terraform/aws/s3/sns
+
+echo "> Deploying infrastructure in AWS with Terraform" && \
+cd $BUILD_DIR/terraform && \
+terraform init && \
+terraform apply \
+-target=module.kms \
+-target=aws_appconfig_application.substation \
+-target=aws_appconfig_environment.example \
+-target=aws_appconfig_deployment_strategy.instant \
+-target=module.ecr
+
+echo "> Building Substation container images and pushing to AWS ECR" && \
+cd $SUBSTATION_ROOT && \
+sh build/scripts/aws/lambda/get_appconfig_extension.sh && \
+docker build --build-arg AWS_ARCHITECTURE=$AWS_ARCHITECTURE -f build/container/aws/lambda/substation/Dockerfile -t substation:latest-$AWS_ARCHITECTURE . && \
+# In production environments the tag should match the version of Substation being deployed
+docker tag substation:latest-$AWS_ARCHITECTURE $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/substation:latest && \
+sh build/scripts/aws/ecr_login.sh && \
+docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/substation:latest
+
+echo "> Deploying Substation nodes with Terraform" && \
+cd $BUILD_DIR/terraform && \
+terraform apply
+
+echo "> Compiling Substation configurations and uploading to AWS AppConfig" && \
+cd $SUBSTATION_ROOT && \
+sh build/scripts/config/compile.sh && \
+SUBSTATION_CONFIG_DIRECTORY=$BUILD_DIR AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_ENVIRONMENT=example AWS_APPCONFIG_DEPLOYMENT_STRATEGY=Instant python3 build/scripts/aws/appconfig/appconfig_upload.py

examples/build/terraform/aws/s3/sns/config/node/config.jsonnet

@@ -0,0 +1,8 @@
+local sub = import '../../../../../../../../build/config/substation.libsonnet';
+
+{
+  concurrency: 1,
+  transforms: [
+    sub.tf.send.stdout(),
+  ],
+}

examples/build/terraform/aws/s3/sns/destroy.sh

@@ -0,0 +1,10 @@
+export AWS_DEFAULT_REGION=$AWS_REGION
+BUILD_DIR=$SUBSTATION_ROOT/examples/build/terraform/aws/s3/sns
+
+echo "> Removing Substation configurations from AWS AppConfig" && \
+cd $SUBSTATION_ROOT && \
+AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_PROFILE_NAME=node python3 build/scripts/aws/appconfig/appconfig_delete.py
+
+echo "> Destroying infrastructure in AWS with Terraform" && \
+cd $BUILD_DIR/terraform && \
+terraform destroy

examples/build/terraform/aws/s3/sns/terraform/_provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  # profile = "default"
+  region = "us-east-1"
+}

examples/build/terraform/aws/s3/sns/terraform/_resources.tf

@@ -0,0 +1,169 @@
+data "aws_caller_identity" "caller" {}
+
+# KMS encryption key that is shared by all Substation resources.
+module "kms" {
+  source = "../../../../../../../build/terraform/aws/kms"
+  config = {
+    name   = "alias/substation"
+    policy = data.aws_iam_policy_document.kms.json
+  }
+}
+
+data "aws_iam_policy_document" "kms" {
+  # Allows CloudWatch to access encrypted resources.
+  statement {
+    sid    = "CloudWatch"
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey"
+    ]
+
+    principals {
+      type        = "Service"
+      identifiers = ["cloudwatch.amazonaws.com"]
+    }
+
+    resources = ["*"]
+  }
+
+  # Allows S3 to access encrypted SNS topic.
+  statement {
+    sid    = "S3"
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey"
+    ]
+
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+
+    resources = ["*"]
+  }
+
+  # Default key policy for KMS.
+  # https://docs.aws.amazon.com/kms/latest/developerguide/determining-access-key-policy.html
+  statement {
+    sid    = "KMS"
+    effect = "Allow"
+    actions = [
+      "kms:*",
+    ]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.caller.account_id}:root"]
+    }
+
+    resources = ["*"]
+  }
+}
+
+# AppConfig application that is shared by all Substation applications.
+resource "aws_appconfig_application" "substation" {
+  name        = "substation"
+  description = "Stores compiled configuration files for Substation"
+}
+
+resource "aws_appconfig_environment" "example" {
+  name           = "example"
+  description    = "Stores example Substation configuration files"
+  application_id = aws_appconfig_application.substation.id
+}
+
+# AWS Lambda requires an instant deployment strategy.
+resource "aws_appconfig_deployment_strategy" "instant" {
+  name                           = "Instant"
+  description                    = "This strategy deploys the configuration to all targets immediately with zero bake time."
+  deployment_duration_in_minutes = 0
+  final_bake_time_in_minutes     = 0
+  growth_factor                  = 100
+  growth_type                    = "LINEAR"
+  replicate_to                   = "NONE"
+}
+
+# Repository for the core Substation application.
+module "ecr" {
+  source = "../../../../../../../build/terraform/aws/ecr"
+  kms    = module.kms
+
+  config = {
+    name         = "substation"
+    force_delete = true
+  }
+}
+
+resource "random_uuid" "s3" {}
+
+# S3 bucket used to store all data.
+module "s3" {
+  source = "../../../../../../../build/terraform/aws/s3"
+  kms    = module.kms
+
+  config = {
+    # Bucket name is randomized to avoid collisions.
+    name = "${random_uuid.s3.result}-substation"
+  }
+
+  access = [
+    module.lambda_node.role.name,
+  ]
+}
+
+module "sns" {
+  source = "../../../../../../../build/terraform/aws/sns"
+  kms    = module.kms
+
+  config = {
+    name = "substation"
+  }
+}
+
+# Grants the S3 bucket permission to publish to the SNS topic.
+resource "aws_sns_topic_policy" "s3_access" {
+  arn    = module.sns.arn
+  policy = data.aws_iam_policy_document.s3_access_policy.json
+}
+
+data "aws_iam_policy_document" "s3_access_policy" {
+  statement {
+    actions = [
+      "sns:Publish",
+    ]
+
+    resources = [
+      module.sns.arn,
+    ]
+
+    condition {
+      test     = "ArnEquals"
+      variable = "aws:SourceArn"
+
+      values = [
+        module.s3.arn,
+      ]
+    }
+
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+
+    effect = "Allow"
+  }
+}
+
+resource "aws_s3_bucket_notification" "sns" {
+  bucket = module.s3.id
+
+  topic {
+    topic_arn = module.sns.arn
+
+    events = [
+      "s3:ObjectCreated:*",
+    ]
+  }
+}

examples/build/terraform/aws/s3/sns/terraform/node.tf

@@ -0,0 +1,38 @@
+module "lambda_node" {
+  source = "../../../../../../../build/terraform/aws/lambda"
+  # These are always required for all Lambda.
+  kms       = module.kms
+  appconfig = aws_appconfig_application.substation
+
+  config = {
+    name        = "node"
+    description = "Substation node that reads data from S3 via SNS."
+    image_uri   = "${module.ecr.url}:latest"
+    image_arm   = true
+
+    env = {
+      "SUBSTATION_CONFIG" : "http://localhost:2772/applications/substation/environments/example/configurations/node"
+      "SUBSTATION_HANDLER" : "AWS_S3_SNS"
+      "SUBSTATION_DEBUG" : true
+    }
+  }
+
+  depends_on = [
+    aws_appconfig_application.substation,
+    module.ecr.url,
+  ]
+}
+
+resource "aws_sns_topic_subscription" "node" {
+  topic_arn = module.sns.arn
+  protocol  = "lambda"
+  endpoint  = module.lambda_node.arn
+}
+
+resource "aws_lambda_permission" "node" {
+  statement_id  = "AllowExecutionFromSNS"
+  action        = "lambda:InvokeFunction"
+  function_name = module.lambda_node.name
+  principal     = "sns.amazonaws.com"
+  source_arn    = module.sns.arn
+}