refactor: Release v1.0.0 (#139)

* build: Go v1.21 * refactor: Use ctx in HTTP Requests * build(terraform): Event Bridge tags * docs: README * refactor(transform): ctrl msg handling in meta transforms * refactor(message): AsControl * feat(transform): Add Unix Millisecond Transforms * ci(go): Update Go Version * fix(transform): Update Factory * ci(go): Go Lint Version * build(terraform): DynamoDB TTL Variable * docs(example): Kinesis Time Travel * style(Python): black Linting * fix(build path): moved config files under /file to /substation dir * fix(transform): Meta Duplicate Control Messages * docs(examples): Time String Conversion * fix(bufio): Dynamic Buffers for AWS Lambda * fix(transform): Dedupe Meta Switch Ctrl Messages * fix(aws): Use No-Cancel Context * fix(bufio): Lambda Buffer Capacity * style(examples): jsonnetfmt * chore(go): Upgrade to 1.22, Dependencies * build(terraform): Optional CMK KMS, AppConfig * build(terraform): KMS ARN * build(terraform): Default Encryption, Refactor S3 WORM * build(terraform): AppConfig, KMS Variable Null Conditions * build(terraform): AppConfig module * build(terraform): Refactor AppConfig Module * docs(examples): Refactor Terraform Examples * style(terraform): Formatting * docs(README): Terraform Example * docs(README): Terraform Example * docs(examples): Fix Lambda Microservice TF * build(config): Unix Milli Transforms * build(config): Fix Source Key * fix(transform): Enrich HTTP Post * feat(aws): Custom Retryer Support * feat(aws): Customizable Retryable AWS Errors * docs(example): XDR S3 Example * build(libsonnet): AWS Secrets * build(aws/s3): S3 Compliance Settings * docs(example): S3 XDR * docs(example): S3 XDR * docs(api): API Docs * docs(md): Repo Docs * fix(deps): Update Go Mod * docs(python): Update Script Doc * build(python): Add Kinesis Put Records Script * style(scripts): black * docs(terraform): Module Docs * build(terraform): Shorten IAM Resource Names * build(dockerfile): Enable CGO * build(terraform): Add AppConfig Validator Support * docs(examples): Add AppConfig Validation Example * build(terraform): Dynamic Validator * docs(examples): AppConfig Validator * docs(examples): AppConfig Lambda * refactor(app): Rename Validation to Validator * build(docker): dockerignore * feat(sub): Support Custom Transforms * chore(internal): Remove Unused Code * examples: Updates --------- Co-authored-by: Mallika Kaur Oberoi <[email protected]>
brexhq · Feb 29, 2024 · 5eb0f4a · 5eb0f4a
1 parent ae81392
commit 5eb0f4a
Show file tree

Hide file tree

Showing 284 changed files with 3,349 additions and 1,306 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/go:1.20
+FROM mcr.microsoft.com/vscode/devcontainers/go:dev-1.22
 RUN apt-get update -y && \
     # https://developer.hashicorp.com/terraform/downloads
     wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg && \

diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,7 @@
+.devcontainer/
+.github/
+.vscode/
+build/config/
+build/container/
+build/terraform/
+examples/
diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml
@@ -16,15 +16,15 @@ jobs:
     - name: setup
       uses: actions/setup-go@v2
       with:
-        go-version: 1.19
+        go-version: 1.22
 
     - name: tests
       run: go test -timeout 30s -v ./... 
 
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.49
+        version: latest
 
   python:
     runs-on: ubuntu-latest
@@ -50,7 +50,7 @@ jobs:
       - name: setup
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: 1.22
 
       - name: compile
         run: |

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
diff --git a/README.md b/README.md
@@ -343,6 +343,7 @@ Substation includes Terraform modules for securely deploying data pipelines and
 ```tcl
 # These resources are deployed once and are used by all Substation infrastructure.
 
+# Substation resources can be encrypted using a customer-managed KMS key.
 module "kms" {
   source = "../../../../../../build/terraform/aws/kms"
 
@@ -351,30 +352,20 @@ module "kms" {
   }
 }
 
-resource "aws_appconfig_application" "substation" {
-  name        = "substation"
-  description = "Stores compiled configuration files for Substation"
-}
-
-resource "aws_appconfig_environment" "example" {
-  name           = "example"
-  description    = "Stores example Substation configuration files"
-  application_id = aws_appconfig_application.substation.id
-}
+# Substation typically uses AppConfig to manage configuration files, but
+# configurations can also be loaded from an S3 URI or an HTTP endpoint.
+module "appconfig" {
+  source = "../../../../../../build/terraform/aws/appconfig"
 
-# AWS Lambda requires an instant deployment strategy.
-resource "aws_appconfig_deployment_strategy" "instant" {
-  name                           = "Instant"
-  description                    = "This strategy deploys the configuration
-  to all targets immediately with zero bake time."
-  deployment_duration_in_minutes = 0
-  final_bake_time_in_minutes     = 0
-  growth_factor                  = 100
-  growth_type                    = "LINEAR"
-  replicate_to                   = "NONE"
+  config = {
+    name = "substation"
+    environments = [{
+        name = "example"
+    }]
+  }
 }
 
-module "ecr_substation" {
+module "ecr" {
   source = "../../../../../../build/terraform/aws/ecr"
   kms    = module.kms
 
@@ -396,10 +387,11 @@ module "s3" {
   }
 
   # Access is granted by providing the role name of a
-  # resource. This access applies least privilege.
+  # resource. This access applies least privilege and
+  # grants access to dependent resources, such as KMS.
   access = [
     # Lambda functions create unique roles that are
-    # used to access to resources.
+    # used to access resources.
     module.node.role.name,
   ]
 }
@@ -424,19 +416,16 @@ module "node_gateway" {
 
 module "node" {
   source = "../../../../../../build/terraform/aws/lambda"
-  # These are always required for all Lambda.
-  kms       = module.kms
-  appconfig = aws_appconfig_application.substation
+  kms       = module.kms  # Optional
+  appconfig = module.appconfig  # Optional
 
   config = {
     name        = "node"
-    description = "Substation node that writes data to S3"
-    image_uri   = "${module.ecr_substation.url}:latest"
+    description = "Substation node that writes data to S3."
+    image_uri   = "${module.ecr.url}:latest"
     image_arm   = true
 
     env = {
-      # Each Lambda includes an AppConfig layer that hosts configuration files locally.
-      # Alternatively, configuration files can be loaded from AWS S3 using an S3 URI.
       "SUBSTATION_CONFIG" : "https://localhost:2772/applications/substation/environments/example/configurations/node"
       # This Substation node will ingest data from API Gateway. More nodes can be 
       # deployed to ingest data from other sources, such as Kinesis or SQS.
@@ -446,8 +435,8 @@ module "node" {
   }
 
   depends_on = [
-    aws_appconfig_application.substation,
-    module.ecr_substation.url,
+    module.appconfig.name,
+    module.ecr.url,
   ]
 }
 ```

diff --git a/VERSIONING.md b/VERSIONING.md
@@ -1,9 +1,8 @@
 # Versioning
 
-Substation uses [Semantic Versioning 2.0](https://semver.org/). Versions are managed using Git tags and are updated by the maintainers when releases are made. The version applies to the [Go module](https://pkg.go.dev/github.com/brexhq/substation) and the components described below:
+Substation uses [Semantic Versioning 2.0](https://semver.org/). Versions are managed using Git tags and are updated by the maintainers when releases are made. The version applies to the [Go module](https://pkg.go.dev/github.com/brexhq/substation) and the components below:
 
 - cmd/aws/*
-- cmd/client/*
 - condition/*
 - config/*
 - message/*

diff --git a/build/config/substation.libsonnet b/build/config/substation.libsonnet
@@ -910,6 +910,14 @@
           type: 'time_from_unix',
           settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))),
         },
+        unix_milli(settings={}): {
+          local default = {
+            object: $.config.object,
+          },
+
+          type: 'time_from_unix_milli',
+          settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))),
+        },
       },
       now(settings={}): {
         local default = {
@@ -940,6 +948,14 @@
         type: 'time_to_unix',
         settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))),
       },
+      unix_milli(settings={}): {
+        local default = {
+          object: $.config.object,
+        },
+
+        type: 'time_to_unix_milli',
+        settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))),
+      },
     },
     util: $.transform.utility,
     utility: {
@@ -1032,24 +1048,26 @@
     metric: { name: null, attributes: null, destination: null },
     object: { source_key: null, target_key: null, batch_key: null },
     request: { timeout: '1s' },
-    retry: { count: 3 },
+    retry: { count: 3, error_messages: null },
   },
   // Mirrors config from the internal/file package.
   file_path: { prefix: null, time_format: '2006/01/02', uuid: true, suffix: null },
   // Mirrors interfaces from the internal/secrets package.
   secrets: {
     default: { id: null, ttl: null },
-    aws_secrets_manager(settings={}): {
-      local default = {
-        id: null,
-        name: null,
-        ttl_offset: null,
-        aws: $.config.aws,
-        retry: $.config.retry,
-      },
+    aws: {
+      secrets_manager(settings={}): {
+        local default = {
+          id: null,
+          name: null,
+          ttl_offset: null,
+          aws: $.config.aws,
+          retry: $.config.retry,
+        },
 
-      type: 'aws_secrets_manager',
-      settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))),
+        type: 'aws_secrets_manager',
+        settings: std.prune(std.mergePatch(default, $.helpers.abbv(settings))),
+      },
     },
     environment_variable(settings={}): {
       local default = { id: null, name: null, ttl_offset: null },
@@ -1063,7 +1081,7 @@
     cnd: $.pattern.condition,
     condition: {
       obj(key): {
-        object: { source: key },
+        object: { source_key: key },
       },
       // Negates any inspector.
       negate(inspector): $.condition.meta.negate(settings={ inspector: inspector }),

diff --git a/build/container/aws/lambda/autoscaling/Dockerfile b/build/container/aws/lambda/autoscaling/Dockerfile
@@ -1,7 +1,8 @@
 ARG AWS_ARCHITECTURE=x86_64
 
-FROM golang:1.21 as build
+FROM golang:1.22 as build
 ENV GO111MODULE=on
+ENV CGO_ENABLED=0
 
 # install AppConfig Lambda extension
 COPY build/scripts/aws/lambda/extension.zip /tmp/extension.zip

diff --git a/build/container/aws/lambda/substation/Dockerfile b/build/container/aws/lambda/substation/Dockerfile
@@ -1,6 +1,6 @@
 ARG AWS_ARCHITECTURE=x86_64
 
-FROM golang:1.21 as build
+FROM golang:1.22 as build
 ENV GO111MODULE=on 
 ENV CGO_ENABLED=0
 

diff --git a/...ontainer/aws/lambda/validation/Dockerfile → ...container/aws/lambda/validator/Dockerfile b/...ontainer/aws/lambda/validation/Dockerfile → ...container/aws/lambda/validator/Dockerfile
@@ -1,7 +1,8 @@
 ARG AWS_ARCHITECTURE=x86_64
 
-FROM golang:1.21 as build
+FROM golang:1.22 as build
 ENV GO111MODULE=on
+ENV CGO_ENABLED=0
 
 # install deps
 WORKDIR /usr/local/go/src/substation/

diff --git a/build/scripts/aws/appconfig/appconfig_delete.py b/build/scripts/aws/appconfig/appconfig_delete.py
@@ -1,11 +1,12 @@
 """Deletes application profiles, including hosted configurations, in AWS AppConfig.
 
-Enumerates hosted configurations in AppConfig for a user-defined application profile and deletes each configuration. When all configurations are deleted, the profile is deleted.
-
-	Typical usage example:
+Enumerates hosted configurations in AppConfig for a user-defined application profile 
+and deletes each configuration. When all configurations are deleted, the profile is deleted.
 
+Usage example:
 	AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_PROFILE_NAME=foo python3 appconfig_delete.py
 """
+
 import os
 
 import boto3

diff --git a/build/scripts/aws/appconfig/appconfig_upload.py b/build/scripts/aws/appconfig/appconfig_upload.py
@@ -1,11 +1,17 @@
 """Manages the upload and deployment of Substation configurations in AWS AppConfig.
 
-Enumerates hosted configurations in AppConfig and retrieves each profile's latest version, uploads local configurations, and deploys configurations if they have changed. AppConfig increments the version number of hosted configurations when new (non-duplicate) content is uploaded, so this script uploads all local configurations and uses the returned version number to determine if deployment is needed. This is intended to be deployed to a CI/CD pipeline (e.g., GitHub Actions, Circle CI, Jenkins, etc.) for automated configuration management.
-
-    Typical usage example:
-
+Enumerates hosted configurations in AppConfig and retrieves each profile's latest 
+version, uploads local configurations, and deploys configurations if they have changed. 
+AppConfig increments the version number of hosted configurations when new (non-duplicate) 
+content is uploaded, so this script uploads all local configurations and uses the returned 
+version number to determine if deployment is needed. This is intended to be deployed to 
+a CI/CD pipeline (e.g., GitHub Actions, Circle CI, Jenkins, etc.) for automated configuration 
+management.
+
+Usage example:
     SUBSTATION_CONFIG_DIRECTORY=examples/aws/config AWS_APPCONFIG_APPLICATION_NAME=substation AWS_APPCONFIG_ENVIRONMENT=prod AWS_APPCONFIG_DEPLOYMENT_STRATEGY=Instant python3 appconfig_upload.py
 """
+
 import json
 import os
 import time

diff --git a/build/scripts/aws/appconfig/appconfig_validate.py b/build/scripts/aws/appconfig/appconfig_validate.py
@@ -2,14 +2,13 @@
 Manages the validation check before deployment of Substation configurations
 in AWS AppConfig.
 
-This is intended to be deployed to a CI/CD
-pipeline (e.g., GitHub Actions, Circle CI, Jenkins, etc.) for automated
-configuration tests.
-
-Typical usage example:
+This is intended to be deployed to a CI/CD pipeline (e.g., GitHub Actions, 
+Circle CI, Jenkins, etc.) for automated configuration tests.
 
+Usage example:
     SUBSTATION_CONFIG_DIRECTORY=config AWS_APPCONFIG_VALIDATOR=substation_validator python3 appconfig_validate.py
 """
+
 import json
 import os
 import base64

diff --git a/build/scripts/aws/kinesis/put_records.py b/build/scripts/aws/kinesis/put_records.py
@@ -0,0 +1,42 @@
+"""
+Puts records into a Kinesis Data Stream.
+
+Usage example:
+    python3 put_records.py my-stream my-file.txt --print-response
+"""
+
+import argparse
+import boto3
+import uuid
+
+CLIENT = boto3.client("kinesis")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Puts records into a Kinesis Data Stream"
+    )
+    parser.add_argument("stream_name", help="The name of the stream")
+    parser.add_argument(
+        "file", help="The file containing data that is put into the stream"
+    )
+    parser.add_argument(
+        "--print-response",
+        help="Determines if the response is printed to the console",
+        action="store_true",
+    )
+    args = parser.parse_args()
+
+    with open(args.file, "rb") as f:
+        for line in f.readlines():
+            data = line.decode("utf-8").strip()
+            resp = CLIENT.put_record(
+                StreamName=args.stream_name, Data=data, PartitionKey=str(uuid.uuid4())
+            )
+
+            if args.print_response:
+                print(resp)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/build/scripts/aws/s3/s3_rehydration.py b/build/scripts/aws/s3/s3_rehydration.py
@@ -2,7 +2,7 @@
 Rehydrates data from an AWS S3 bucket into an SNS topic by simulating S3 
 object creation events.
 
-Typical usage example:
+Usage example:
 
     python3 s3_rehydration.py --bucket my-bucket --topic my-topic 
     --prefix my-prefix --filter my-filter my-other-filter