chore: GHA to smoke test against last released exemplar (#2983)

This adds a GitHub Action that smoke tests branches against the exemplar
in the latest tagged release.

It will fetch the latest tagged FTL release from dockerhub, checkout the
repo at that tag, then deploy that version to a local kube cluster.
Testing this tagged version against the current exemplar is disabled
(chicken-and-egg since test change are needed in the tagged release).
Finally, it will upgrade the local kube cluster to that in the current
branch and run the exemplar smoketest to verify the upgrade path was
successful.

---------

Co-authored-by: Stuart Douglas <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/smoketest.yml

@@ -0,0 +1,92 @@
+on:
+  push:
+    branches:
+      - main
+name: SmokeTest
+concurrency:
+  group: ${{ github.ref }}-smoketest
+  cancel-in-progress: true
+jobs:
+  smoke-test-upgrade-path:
+    name: Smoke Test Upgrade Path
+    if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository with full history
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Get last tagged release
+        run: |
+          latest_release=$(git tag --sort=-v:refname | grep -v v1 | head -n 1)
+          echo "Latest release: $latest_release"
+          echo "LATEST_VERSION=${latest_release#v}" >> "$GITHUB_ENV"
+      - name: Check out the last tagged release
+        uses: actions/checkout@v4
+        with:
+          ref: v${{ env.LATEST_VERSION }}
+          fetch-depth: 1
+          fetch-tags: true
+      # The following grabs the current smoke test and deployment Justfile from the HEAD. This
+      # is to solve the chicken-and-egg. Once it's in the tagged release, skip this.
+      - name: Check out the current smoke test
+        uses: actions/checkout@v4
+        with:
+          clean: false
+          repository: TBD54566975/ftl.git
+          path: tmpsmoketest
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+          sparse-checkout: |
+            smoketest
+            deployment
+          sparse-checkout-cone-mode: false
+      # When we remove the above step to fetch the current smoke tests and deployment Justfile
+      # into tmpsmoketest, remove the overwrite step as well.
+      - name: Replace the tagged release smoketest with the current smoketest
+        run: |
+          set -euo pipefail
+          echo "Replacing tagged release smoketest with current smoketest"
+          rm -r ./smoketest && mv ./tmpsmoketest/smoketest ./smoketest
+          echo "Replacing deployment Justfile with current Justfile"
+          cp ./tmpsmoketest/deployment/Justfile ./deployment/Justfile
+          echo "Removing tmpsmoketest"
+          rm -r ./tmpsmoketest
+      - name: Init Hermit
+        uses: cashapp/activate-hermit@v1
+        with:
+          cache: true
+      - name: Build Cache
+        uses: ./.github/actions/build-cache
+      - name: Download Go Modules
+        run: go mod download
+      - name: Set up a kube cluster with the tagged release dockerhub image
+        run: |
+            set -euo pipefail
+            echo "Deploying the tagged release to the cluster"
+            cd deployment && just deploy-version ${{ env.LATEST_VERSION }} && cd ..
+      # We skip this since it requires an integration test change to skip the full kube deploy.
+      # Re-enable this step when the tagged release has the integration test change.
+      # - name: Smoke test the tagged release images
+      #   run: |
+      #       set -euo pipefail
+      #       echo "Running smoke test on the tagged release images"
+      #       USE_DB_CONFIG=true SKIP_KUBE_FULL_DEPLOY=true go test -v -tags smoketest -run '^Test' ./smoketest
+      - name: Check out HEAD of the current branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          fetch-depth: 1
+      - name: Smoke test HEAD with a full deploy to test upgrade path
+        run: |
+            set -euo pipefail
+            echo "Running smoke test on the HEAD images"
+            USE_DB_CONFIG=true go test -v -timeout 15m -tags smoketest -run '^Test' ./smoketest
+      - name: Archive Report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: kube-report-smoke-test-upgrade
+          path: /tmp/ftl-kube-report/

deployment/Justfile

@@ -72,8 +72,9 @@ apply:
   kubectl delete job --ignore-not-found=true ftl-dbmig-latest
   helm upgrade --install ftl ../charts/ftl -f values.yaml
 
-deploy-version version: setup-cluster
+deploy-version version: setup-cluster install-istio
   helm repo add ftl https://tbd54566975.github.io/ftl-charts --force-update
+  helm upgrade --install ftl ftl/ftl --version={{version}} -f values-release.yaml || sleep 5 # wait for CRDs to be created, the initial apply will usually fail
   helm upgrade --install ftl ftl/ftl --version={{version}} -f values-release.yaml
 
 delete:

internal/integration/actions.go

@@ -1,4 +1,4 @@
-//go:build integration || infrastructure
+//go:build integration || infrastructure || smoketest
 
 package integration
 

internal/integration/harness.go

@@ -1,4 +1,4 @@
-//go:build integration || infrastructure
+//go:build integration || infrastructure || smoketest
 
 package integration
 

smoketest/echo/echo.go

@@ -0,0 +1,14 @@
+// This is the echo module.
+package echo
+
+import (
+	"context"
+	"fmt"
+)
+
+// Echo returns a greeting with the current time.
+//
+//ftl:verb export
+func Echo(ctx context.Context, req string) (string, error) {
+	return fmt.Sprintf("Hello, %s!!!", req), nil
+}

smoketest/echo/ftl.toml

@@ -0,0 +1,2 @@
+module = "echo"
+language = "go"

smoketest/echo/go.mod

@@ -0,0 +1,18 @@
+module ftl/echo
+
+go 1.23.0
+
+replace github.com/TBD54566975/ftl => ../..
+
+require github.com/TBD54566975/ftl v0.0.0-00010101000000-000000000000
+
+require (
+	github.com/alecthomas/participle/v2 v2.1.1 // indirect
+	github.com/alecthomas/types v0.16.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
+	github.com/hashicorp/cronexpr v1.1.2 // indirect
+	github.com/swaggest/jsonschema-go v0.3.72 // indirect
+	github.com/swaggest/refl v1.3.0 // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
+)