build(terraform): Add CloudWatch Modules

build/terraform/aws/README.md

@@ -26,6 +26,16 @@ This module creates an API Gateway that sends a record to a Kinesis Data Stream.
 
 This module creates an API Gateway that invokes and sends a record to a Lambda function.
 
+### CloudWatch
+
+#### Destination
+
+This module creates a CloudWatch Logs destination that can be used to receive logs from any AWS account or region and send them to a destination.
+
+#### Subscription
+
+This module creates a CloudWatch Logs subscription filter that can be used to send logs from a CloudWatch Logs group to a destination. Use this with the `Destination` module to send logs from any AWS account or region to a single destination.
+
 ### DynamoDB
 
 This module is used as a template for deploying new DynamoDB tables with autoscaling enabled. These tables have [time to live](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/TTL.html) enabled and users can optionally use it by writing values to the `ttl` column.

build/terraform/aws/cloudwatch/destination/_variables.tf

@@ -0,0 +1,22 @@
+variable "kms" {
+  type = object({
+    arn = string
+    id  = string
+  })
+  description = "KMS key used to encrypt the resources."
+}
+
+variable "config" {
+  type = object({
+    name            = string
+    destination_arn = string
+    account_ids     = optional(list(string), [])
+  })
+
+  description = "Configuration for the CloudWatch destination."
+}
+
+variable "tags" {
+  type    = map(any)
+  default = {}
+}

build/terraform/aws/cloudwatch/destination/_versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = "~> 1.5"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}

build/terraform/aws/cloudwatch/destination/main.tf

@@ -0,0 +1,129 @@
+data "aws_region" "current" {}
+
+data "aws_caller_identity" "current" {}
+
+locals {
+  # By default, the current account is included in the list of accounts.
+  account_ids = concat(var.config.account_ids, [data.aws_caller_identity.current.account_id])
+}
+
+data "aws_iam_policy_document" "destination_assume_role" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+    principals {
+      type = "Service"
+      identifiers = [
+        "logs.amazonaws.com"
+      ]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:SourceArn"
+
+      # Creates a list of wildcarded ARNs for each account.
+      values = formatlist("arn:aws:logs:*:%s:*", local.account_ids)
+    }
+  }
+}
+
+data "aws_iam_policy_document" "destination" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey"
+    ]
+
+    // Access the KMS key.
+    resources = [
+      var.kms.arn,
+    ]
+  }
+
+  // If the destination is Kinesis Firehose, the role must have write access.
+  dynamic "statement" {
+    for_each = strcontains(var.config.destination_arn, "arn:aws:firehose:") ? [1] : []
+
+    content {
+      effect = "Allow"
+      actions = [
+        "firehose:DescribeDeliveryStream",
+        "firehose:PutRecord",
+        "firehose:PutRecordBatch",
+      ]
+
+      resources = [
+        var.config.destination_arn,
+      ]
+    }
+  }
+
+  // If the destination is Kinesis Data Stream, the role must have write access.
+  dynamic "statement" {
+    for_each = strcontains(var.config.destination_arn, "arn:aws:kinesis:") ? [1] : []
+
+    content {
+      effect = "Allow"
+      actions = [
+        "kinesis:DescribeStream",
+        "kinesis:DescribeStreamSummary",
+        "kinesis:DescribeStreamConsumer",
+        "kinesis:SubscribeToShard",
+        "kinesis:RegisterStreamConsumer",
+        "kinesis:PutRecord",
+        "kinesis:PutRecords",
+      ]
+
+      resources = [
+        var.config.destination_arn,
+      ]
+    }
+  }
+}
+
+resource "aws_iam_role" "destination" {
+  name               = "sub-cloudwatch-destination-${var.config.name}-${data.aws_region.current.name}"
+  assume_role_policy = data.aws_iam_policy_document.destination_assume_role.json
+  tags               = var.tags
+}
+
+resource "aws_iam_role_policy_attachment" "destination" {
+  role       = aws_iam_role.destination.name
+  policy_arn = aws_iam_policy.destination.arn
+}
+
+resource "aws_iam_policy" "destination" {
+  name        = "sub-cloudwatch-destination-${var.config.name}-${data.aws_region.current.name}"
+  description = "Policy for the ${var.config.name} CloudWatch destination."
+  policy      = data.aws_iam_policy_document.destination.json
+}
+
+resource "aws_cloudwatch_log_destination" "destination" {
+  name       = var.config.name
+  role_arn   = aws_iam_role.destination.arn
+  target_arn = var.config.destination_arn
+}
+
+data "aws_iam_policy_document" "destination_access" {
+  statement {
+    effect = "Allow"
+    principals {
+      type        = "AWS"
+      identifiers = local.account_ids
+    }
+
+    actions = [
+      "logs:PutSubscriptionFilter",
+    ]
+    resources = [
+      aws_cloudwatch_log_destination.destination.arn,
+    ]
+  }
+}
+
+resource "aws_cloudwatch_log_destination_policy" "destination" {
+  destination_name = aws_cloudwatch_log_destination.destination.name
+  access_policy    = data.aws_iam_policy_document.destination_access.json
+}

build/terraform/aws/cloudwatch/destination/output.tf

@@ -0,0 +1,7 @@
+output "role" {
+  value = aws_iam_role.destination
+}
+
+output "arn" {
+  value = aws_cloudwatch_log_destination.destination.arn
+}

build/terraform/aws/cloudwatch/subscription/_variables.tf

@@ -0,0 +1,16 @@
+variable "config" {
+  type = object({
+    name            = string
+    destination_arn = string
+    log_groups      = list(string)
+    filter_pattern  = optional(string, "")
+
+  })
+
+  description = "Configuration for the CloudWatch subscription filter."
+}
+
+variable "tags" {
+  type    = map(any)
+  default = {}
+}

build/terraform/aws/cloudwatch/subscription/_versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = "~> 1.2"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}

build/terraform/aws/cloudwatch/subscription/main.tf

@@ -0,0 +1,12 @@
+resource "aws_cloudwatch_log_subscription_filter" "subscription_filter" {
+  for_each       = toset(var.config.log_groups)
+  log_group_name = each.key
+
+  name            = var.config.name
+  destination_arn = var.config.destination_arn
+  # By default there is no filter pattern, so all logs are sent to the destination.
+  filter_pattern = var.config.filter_pattern
+
+  # If the destination is a Kinesis stream, then randomly distribute the logs to avoid hot shards.
+  distribution = "Random"
+}

examples/build/terraform/aws/README.md

@@ -2,6 +2,64 @@
 
 These example deployments demonstrate different use cases for Substation on AWS.
 
+# CloudWatch Logs
+
+## Cross-Account / Cross-Region
+
+Deploys a data pipeline that collects data from CloudWatch log groups in any account or region into a Kinesis Data Stream.
+
+```mermaid
+
+flowchart LR
+    %% resources
+    cw1([CloudWatch Log Group])
+    cw2([CloudWatch Log Group])
+    cw3([CloudWatch Log Group])
+    kds([Kinesis Data Stream])
+
+    consumerHandler[[Handler]]
+    consumerTransforms[Transforms]
+
+    subgraph Account B / Region us-west-2
+    cw2
+    end
+
+    subgraph Account A / Region us-west-2
+    cw3
+    end
+
+    subgraph Account A / Region us-east-1
+    cw1 --> kds
+    cw3 --> kds
+    cw2 --> kds
+    kds --> consumerHandler
+
+    subgraph Substation Consumer Node 
+    consumerHandler  --> consumerTransforms
+    end
+    end
+```
+
+## To Lambda
+
+Deploys a data pipeline that sends data from a CloudWatch log group to a Lambda function.
+
+```mermaid
+
+flowchart LR
+    %% resources
+    cw([CloudWatch Log Group])
+
+    consumerHandler[[Handler]]
+    consumerTransforms[Transforms]
+
+    cw --> consumerHandler
+
+    subgraph Substation Consumer Node 
+    consumerHandler  --> consumerTransforms
+    end
+```
+
 # DynamoDB
 
 ## Change Data Capture (CDC)
@@ -60,26 +118,31 @@ Deploys a data pipeline that implements a multi-phase streaming data pattern usi
 
 flowchart LR
     %% resources
-    gateway([API Gateway])
-    kds1([Kinesis Data Stream])
-    kds2([Kinesis Data Stream])
+    cw1([CloudWatch Log Group])
+    cw2([CloudWatch Log Group])
+    cw3([CloudWatch Log Group])
+    kds([Kinesis Data Stream])
 
-    publisherHandler[[Handler]]
-    publisherTransforms[Transforms]
+    consumerHandler[[Handler]]
+    consumerTransforms[Transforms]
 
-    subscriberHandler[[Handler]]
-    subscriberTransforms[Transforms]
+    subgraph Account B / Region us-west-2
+    cw2
+    end
 
-    %% connections
-    gateway --> kds1 --> publisherHandler
-    subgraph Substation Publisher Node 
-    publisherHandler --> publisherTransforms
+    subgraph Account A / Region us-west-2
+    cw3
     end
 
-    publisherTransforms --> kds2 --> subscriberHandler
+    subgraph Account A / Region us-east-1
+    cw1 --> kds
+    cw3 --> kds
+    cw2 --> kds
+    kds --> consumerHandler
 
-    subgraph Substation Subscriber Node 
-    subscriberHandler  --> subscriberTransforms
+    subgraph Substation Consumer Node 
+    consumerHandler  --> consumerTransforms
+    end
     end
 ```