Validation framework for terraform examples (#250)

* Validation framework * Add explicit dependency for FARGATE * Fmt * Make examples non flaky * Add sameness * More fixes * Pivoted to a registry based framework * Moved all scenarios to use the registry pattern * Add validations for scenario registration * Efficiently parse terraform outputs * Update README * Add registry tests * Panic on registration errors
hashicorp · Dec 20, 2023 · 3da9a18 · 3da9a18
1 parent 11cb4ae
commit 3da9a18
Show file tree

Hide file tree

Showing 23 changed files with 1,957 additions and 4 deletions.
diff --git a/.github/workflows/nightly-ecs-examples-validator.yml b/.github/workflows/nightly-ecs-examples-validator.yml
@@ -0,0 +1,115 @@
+name: Nighly ECS example validator
+on:
+  workflow_dispatch:
+
+jobs:
+  get-go-version:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./test/acceptance
+    outputs:
+      go-version: ${{ steps.get-go-version.outputs.go-version }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: Determine Go version
+      id: get-go-version
+      run: |
+        echo "Building with Go $(cat .go-version)"
+        echo "go-version=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+  go-fmt-and-lint-acceptance:
+    runs-on: ubuntu-latest
+    needs:
+      - get-go-version
+    defaults:
+      run:
+        working-directory: ./test/acceptance
+    steps:
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: Setup Go
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: ${{ needs.get-go-version.outputs.go-version }}
+        cache-dependency-path: ./test/acceptance/go.sum
+    - name: Go CI lint
+      uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
+      with:
+        args: "--verbose --enable gofmt"
+        only-new-issues: false
+        skip-pkg-cache: true
+        skip-build-cache: true
+        working-directory: ./test/acceptance
+    - name: Lint Consul retry
+      run: |
+        go install github.com/hashicorp/[email protected]
+        lint-consul-retry
+  terraform-fmt:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v3
+      with:
+        terraform_version: 1.4.2
+    - name: Validate
+      run: terraform fmt -check -recursive .
+  single-cluster:
+    needs:
+    - terraform-fmt
+    - go-fmt-and-lint-acceptance
+    - get-go-version
+    strategy:
+      matrix:
+        name:
+          - Consul ECS on Fargate
+          - Consul ECS on EC2
+          - Consul ECS with HCP
+        include:
+          - name: Consul ECS on Fargate
+            scenario: FARGATE
+
+          - name: Consul ECS on EC2
+            scenario: EC2
+
+          - name: Consul ECS with HCP
+            scenario: HCP
+      fail-fast: false
+    uses: ./.github/workflows/reusable-ecs-example-validator.yml
+    with:
+      name: ${{ matrix.name }}
+      scenario: ${{ matrix.scenario }}
+      go-version: ${{ needs.get-go-version.outputs.go-version }}
+    secrets: inherit
+  multi-cluster:
+    needs:
+    - single-cluster
+    - get-go-version
+    strategy:
+      matrix:
+        name:
+        - Cluster Peering
+        - WAN Federation with Mesh gateways
+        - Locality Aware Routing
+        - Service Sameness
+        include:
+        - name: Cluster Peering
+          scenario: CLUSTER_PEERING
+
+        - name: WAN Federation with Mesh gateways
+          scenario: WAN_FEDERATION
+
+        - name: Locality Aware Routing
+          scenario: LOCALITY_AWARE_ROUTING
+
+        - name: Service Sameness
+          scenario: SERVICE_SAMENESS
+      fail-fast: false
+    uses: ./.github/workflows/reusable-ecs-example-validator.yml
+    with:
+      name: ${{ matrix.name }}
+      scenario: ${{ matrix.scenario }}
+      go-version: ${{ needs.get-go-version.outputs.go-version }}
+    secrets: inherit
diff --git a/.github/workflows/reusable-ecs-example-validator.yml b/.github/workflows/reusable-ecs-example-validator.yml
@@ -0,0 +1,74 @@
+name: reusable-ecs-example-validator
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        description: "The name of the job"
+        required: true
+        type: string        
+      scenario:
+        description: "The name of the scenario that needs to be tested on ECS. This will be passed as an environment variable to the test."
+        required: true
+        type: string
+      go-version:
+        description: "Version of Go to use to run the tests"
+        required: true
+        type: string
+
+env:
+  TEST_RESULTS: /tmp/test-results
+  GOTESTSUM_VERSION: 1.8.0
+  CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }}
+  HCP_CLIENT_ID: ${{ secrets.HCP_CLIENT_ID }}
+  HCP_CLIENT_SECRET: ${{ secrets.HCP_CLIENT_SECRET }}
+  HCP_PROJECT_ID: ${{ secrets.HCP_PROJECT_ID }}
+  TEST_SCENARIO: ${{ inputs.scenario }}
+
+jobs:
+  example-validator:
+    name: ${{ inputs.name }}
+    runs-on: ['ubuntu-latest']
+    defaults:
+      run:
+        working-directory: ./test/acceptance/examples
+    steps:
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: Setup Go
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: ${{ inputs.go-version }}
+        cache-dependency-path: ./test/acceptance/go.sum
+    - name: Install gotestsum
+      run: |
+        curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v${{ env.GOTESTSUM_VERSION }}/gotestsum_${{ env.GOTESTSUM_VERSION }}_linux_amd64.tar.gz" | \
+        tar -xz --overwrite -C /usr/local/bin gotestsum
+    - name: Install AWS CLI
+      run: |
+        curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
+        sudo dpkg -i session-manager-plugin.deb
+        aws --version
+        echo session-manager-plugin version "$(session-manager-plugin --version)"
+    - name: Install AWS ECS CLI
+      run: |
+        curl -sSL "https://amazon-ecs-cli.s3.amazonaws.com/ecs-cli-linux-amd64-latest" -o /usr/local/bin/ecs-cli
+        chmod +x /usr/local/bin/ecs-cli
+        ecs-cli --version
+    - name: Assume AWS IAM Role
+      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      with:
+        role-to-assume: ${{ secrets.AWS_ECS_ROLE_ARN }}
+        aws-region: ${{ secrets.AWS_ECS_REGION }}
+        aws-access-key-id: ${{ secrets.AWS_ECS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_ECS_SECRET_ACCESS_KEY }}
+        role-duration-seconds: 7200
+    - name: Validation
+      run: |
+        mkdir -p "$TEST_RESULTS"
+        gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" --format standard-verbose -- ./... -p 1 -timeout 1h -v -failfast
+    - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+      if: always()
+      with:
+        name: acceptance-test-results
+        path: ${{ env.TEST_RESULTS }}/gotestsum-report.xml
diff --git a/examples/dev-server-fargate/main.tf b/examples/dev-server-fargate/main.tf
@@ -50,6 +50,8 @@ resource "aws_ecs_service" "example_client_app" {
 }
 
 module "example_client_app" {
+  depends_on = [module.dev_consul_server]
+
   source = "../../modules/mesh-task"
   family = "${var.name}-example-client-app"
   port   = "9090"
@@ -112,6 +114,8 @@ resource "aws_ecs_service" "example_server_app" {
 }
 
 module "example_server_app" {
+  depends_on = [module.dev_consul_server]
+
   source            = "../../modules/mesh-task"
   family            = "${var.name}-example-server-app"
   port              = "9090"

diff --git a/examples/locality-aware-routing/README.md b/examples/locality-aware-routing/README.md
@@ -79,6 +79,7 @@ Changes to Outputs:
   + client_lb_address             = (known after apply)
   + consul_server_bootstrap_token = (sensitive value)
   + consul_server_url             = (known after apply)
+  + ecs_cluster_arn = (known after apply)
 ```
 
 Type `yes` to apply the changes.
@@ -97,6 +98,7 @@ Outputs:
 client_lb_address = "http://example-client-app-1959503271.us-west-2.elb.amazonaws.com:9090/ui"
 consul_server_bootstrap_token = <sensitive>
 consul_server_url = "http://ecs-dc1-consul-server-713584774.us-west-2.elb.amazonaws.com:8500"
+ecs_cluster_arn = "arn:aws:ecs:us-east-1:123456789012:cluster/my-ecs-cluster"
 ```
 
 ### Explore
@@ -115,7 +117,7 @@ to view the Consul UI and log in using the `consul_server_bootstrap_token` above
 
 If you browse to the URL of the `client_lb_address`, the example application UI should be displayed:
 
-![Example App UI](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/_docs/locality-aware-dc1-ui.png?raw=true)
+![Example App UI](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/_docs/locality-aware-app-ui.png?raw=true)
 
 Notice the IP of the upstream server application's task. Because of the locality parameters added during the service registration, Consul takes care of routing traffic from the client application to the server application task within the same availability zone. You can read more about the locality aware routing feature [here](https://developer.hashicorp.com/consul/docs/v1.17.x/connect/manage-traffic/route-to-local-upstreams?ajs_aid=54615e8b-87b1-40fa-aecc-3e16280d6a88&product_intent=consul)
 

diff --git a/examples/locality-aware-routing/outputs.tf b/examples/locality-aware-routing/outputs.tf
@@ -12,4 +12,8 @@ output "consul_server_bootstrap_token" {
 
 output "client_lb_address" {
   value = "http://${aws_lb.example_client_app.dns_name}:9090/ui"
+}
+
+output "ecs_cluster_arn" {
+  value = module.cluster.ecs_cluster.arn
 }
diff --git a/test/acceptance/README.md b/test/acceptance/README.md
@@ -1,6 +1,6 @@
 ## Acceptance Tests
 
-These tests run the Terraform code.
+These tests run the Terraform code. For scenario based tests please refer to [this](./examples/README.md) document.
 
 ### Prerequisites
 
@@ -27,9 +27,9 @@ The following prerequisites are needed to run the acceptance tests:
 1. Now you can run the tests. The tests accept flags for passing in information about the
    VPC and ECS cluster you've just created.
 
-   Switch back to the `test/acceptance` directory:
+   Switch back to the `test/acceptance/tests` directory:
 
-1. To run the tests, use `go test` from the `test/acceptance` directory:
+1. To run the tests, use `go test` from the `test/acceptance/tests` directory:
 
    For tests that use Consul Enterprise outside of HCP, you must set the
    `CONSUL_LICENSE` environment variable to a Consul Enterprise license key.

diff --git a/test/acceptance/examples/README.md b/test/acceptance/examples/README.md
@@ -0,0 +1,45 @@
+## Scenario based tests
+
+These tests deploy the terraform code present under the `examples/` folder and performs custom validations on the same.
+
+These tests are run as part of [CI](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/.github/workflows/nightly-ecs-examples-validator.yml). The workflow is setup in a way that deployments happen in multiple stages one after the other. We made a conscious choice to run atmost 4 parallel deployments/test jobs in the CI to make sure that we don't exceed the VPC limits set up in the target AWS account.
+
+### Prerequisites
+
+The following prerequisites are needed to run the acceptance tests:
+
+- [Go](https://go.dev/dl/) (`go test`)
+- [Terraform](https://www.terraform.io/downloads) (`terraform`)
+   - [Authentication for AWS provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication)
+   - [Authentication for HCP provider](https://registry.terraform.io/providers/hashicorp/hcp/latest/docs/guides/auth)
+- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) (`aws`)
+   - [AWS Session Manager Plugin](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html)
+- [Amazon ECS CLI](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_CLI_installation.html) (`ecs-cli`)
+
+### Instructions
+
+1. Make sure to set relevant environment variables to configure AWS and HCP (if you are running HCP based scenarios) credentials.
+
+1. Make sure to set the `TEST_SCENARIO` environment variable. This must match one of the scenarios listed in the `scenarioFuncs` map present in [main.go](./main_test.go).
+
+1. To run the tests, use `go test` from the `test/acceptance/examples` directory:
+
+   For tests that use Consul Enterprise outside of HCP, you must set the
+   `CONSUL_LICENSE` environment variable to a Consul Enterprise license key.
+
+   ```sh
+   export CONSUL_LICENSE=$(cat path/to/license-file)
+   TEST_SCENARIO=EC2 go test -run TestScenario -p 1 -timeout 30m -v
+   ```
+
+   You may want to set the `NO_CLEANUP_ON_FAILURE` environment variable if you're debugging
+   a failing test. Without this variable, the tests will delete all resources
+   regardless of passing or failing.
+
+### Adding a new scenario
+
+1. We expect every scenario to register itself to the scenario registry. Similar to existing examples, add a new folder corresponding to your scenario under the `scenarios/` subfolder and add relevant code into the same.
+
+1. Make sure to call the function that adds the scenario to the registry from [main_test.go](./main_test.go).
+
+1. If you want your scenario to run as part of CI, make sure to add it to the matrix list in [this](https://github.com/hashicorp/terraform-aws-consul-ecs/blob/main/.github/workflows/nightly-ecs-examples-validator.yml) workflow file. If the number of parallel jobs within a matrix exceeds 4, make sure to create a new matrix job that is dependent on the existing ones and add your scenario there.