Begin work on testing

backend/controller/scaling/k8sscaling/deployment_provisioner.go

@@ -284,7 +284,7 @@ func (r *DeploymentProvisioner) syncDeployment(ctx context.Context, thisImage st
 			deployment.Spec.Replicas = &ftlDeployment.Runtime.MinReplicas
 		})
 	}
-	changes = r.updateEnvVar(deployment, "FTL_DEPLOYMENT_NAME", deployment.Name, changes)
+	changes = r.updateEnvVar(deployment, "FTL_DEPLOYMENT", deployment.Name, changes)
 	changes = r.updateEnvVar(deployment, "FTL_ENDPOINT", r.FTLEndpoint, changes)
 	return changes, nil
 }

backend/controller/scaling/k8sscaling/kube_scaling_integration_test.go

@@ -0,0 +1,22 @@
+//go:build integration
+
+package k8sscaling_test
+
+import (
+	"testing"
+
+	"github.com/alecthomas/assert/v2"
+
+	in "github.com/TBD54566975/ftl/internal/integration"
+)
+
+func TestKubeScaling(t *testing.T) {
+	in.Run(t,
+		in.WithKubernetes(),
+		in.CopyModule("echo"),
+		in.Deploy("echo"),
+		in.Call("echo", "echo", "Bob", func(t testing.TB, response string) {
+			assert.Equal(t, "Hello, Bob!!!", response)
+		}),
+	)
+}

backend/controller/scaling/k8sscaling/testdata/go/echo/echo.go

@@ -0,0 +1,14 @@
+// This is the echo module.
+package echo
+
+import (
+	"context"
+	"fmt"
+)
+
+// Echo returns a greeting with the current time.
+//
+//ftl:verb export
+func Echo(ctx context.Context, req string) (string, error) {
+	return fmt.Sprintf("Hello, %s!!!", req), nil
+}

backend/controller/scaling/k8sscaling/testdata/go/echo/ftl.toml

@@ -0,0 +1,2 @@
+module = "echo"
+language = "go"

backend/controller/scaling/k8sscaling/testdata/go/echo/go.mod

@@ -0,0 +1,5 @@
+module ftl/echo
+
+go 1.23.0
+
+replace github.com/TBD54566975/ftl => ./../../../../../../..

backend/runner/runner.go

@@ -225,7 +225,7 @@ func (s *Service) Ping(ctx context.Context, req *connect.Request[ftlv1.PingReque
 	return connect.NewResponse(&ftlv1.PingResponse{}), nil
 }
 
-func (s *Service) Deploy(ctx context.Context) error {
+func (s *Service) deploy(ctx context.Context) error {
 	logger := log.FromContext(ctx)
 	if err, ok := s.registrationFailure.Load().Get(); ok {
 		observability.Deployment.Failure(ctx, optional.None[string]())
@@ -392,7 +392,7 @@ func (s *Service) registrationLoop(ctx context.Context, send func(request *ftlv1
 		s.state.Store(state)
 	}
 
-	logger.Tracef("Registering with Controller as %s", state)
+	logger.Infof("Registering with Controller as %s for deployment %s", state, s.config.Deployment)
 	err := send(&ftlv1.RegisterRunnerRequest{
 		Key:        s.key.String(),
 		Endpoint:   s.config.Advertise.String(),
@@ -406,7 +406,7 @@ func (s *Service) registrationLoop(ctx context.Context, send func(request *ftlv1
 		return fmt.Errorf("failed to register with Controller: %w", err)
 	}
 	if state == ftlv1.RunnerState_RUNNER_NEW {
-		err = s.Deploy(ctx)
+		err = s.deploy(ctx)
 		if err != nil {
 			// Deployment failed, we will retry on the next heartbeat
 			// We rely on Deploy to cancel the context if the error is terminal

deployment/Dockerfile.runner.test

@@ -14,4 +14,4 @@ ENV FTL_ENDPOINT="http://host.docker.internal:8892"
 ENV FTL_RUNNER_BIND="http://0.0.0.0:8893"
 ENV FTL_RUNNER_ADVERTISE="http://127.0.0.1:8893"
 
-CMD ["/root/ftl-runner", "--template-dir=template", "--deployment-dir=deployments"]
+CMD ["/root/ftl-runner",  "--deployment-dir=deployments"]

deployment/Justfile

@@ -17,26 +17,40 @@ start: setup full-deploy
 
 rm: teardown
 
-full-deploy:
-  just build-controller
-  just build-runner
+full-deploy: build-controller build-runner setup-cluster
+  #!/bin/bash
+  kubectl rollout restart deployment ftl-controller || true # if this exists already restart it to get the latest image
   just apply || sleep 5 # wait for CRDs to be created, the initial apply will usually fail
   just apply
-  kubectl wait --for=condition=available deployment/ftl-controller --timeout=5m
-  kubectl wait --for=condition=available deployment/ftl-runner --timeout=5m
-  kubectl wait --for=condition=ready pod/ftl-pg-cluster-1-0 --timeout=5m
 
+wait-for-kube:
+  #!/bin/bash
+  while [ -z "$(kubectl get pod ftl-pg-cluster-1-0)" ]; do sleep 1; done
+  kubectl wait --for=condition=ready pod/ftl-pg-cluster-1-0 --timeout=5m
+  kubectl wait --for=condition=available deployment/ftl-controller --timeout=5m
+  sleep 1
   ftl status
 
+
 setup-registry:
+  #!/bin/bash
+  if [ -z "$(k3d registry list | grep {{registry_short}})" ]; then
   k3d registry create {{registry_short}} --port 5000
+  fi
 
-setup-cluster:
-  k3d cluster create ftl --api-port 6550 -p "8992:80@loadbalancer"  --agents 2 \
+setup-cluster: setup-registry
+  #!/bin/bash
+  if [ -z "$(k3d cluster list | grep ftl)" ]; then
+  k3d cluster create ftl --api-port 6550 -p "8892:80@loadbalancer"  --agents 2 \
       --registry-use {{registry_full}} \
-      --registry-config '{{mirrors}}'
+      --registry-config '{{mirrors}}' && \  # Start installing the DB for performance reasons
+      kubectl apply -k base/db-create || sleep 5 && \ # wait for CRDs to be created, the initial apply will usually fail
+      kubectl apply -k base/db-create & # background the process, so we can do other things and speed up the test
+  fi
   kubectl config set-context --current --namespace=default
 
+
+
 setup: setup-registry setup-cluster
 
 teardown-registry:
@@ -84,13 +98,13 @@ build-executables:
   cd ../ && GOARCH=amd64 GOOS=linux CGO_ENABLED=0 just build ftl-controller ftl-runner ftl-initdb ftl
   cp ../build/release/* ./docker-build/
 
-build-controller: build-executables
+build-controller: build-executables setup-registry setup-cluster
   docker build --platform linux/amd64 -t ftl-controller:latest -f Dockerfile.controller.test .
   docker tag ftl-controller:latest {{registry_local}}/ftl-controller:latest
   docker push {{registry_local}}/ftl-controller:latest
 
-build-runner: build-executables
-  docker build --platform linux/amd64 -t ftl-controller:latest -f Dockerfile.runner.test .
+build-runner: build-executables setup-registry setup-cluster
+  docker build --platform linux/amd64 -t ftl-runner:latest -f Dockerfile.runner.test .
   docker tag ftl-runner:latest {{registry_local}}/ftl-runner:latest
   docker push {{registry_local}}/ftl-runner:latest
 

deployment/base/db-create/kustomization.yml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - https://raw.githubusercontent.com/reactive-tech/kubegres/v1.18/kubegres.yaml
   - pg-cluster.yaml

deployment/base/db-migrate/db-migrate.yml

@@ -24,4 +24,4 @@ spec:
             name: ftl-db-migrate-config
   parallelism: 1
   backoffLimit: 1000
-  ttlSecondsAfterFinished: 300
+  ttlSecondsAfterFinished: 1

deployment/base/db-migrate/kustomization.yml

@@ -20,3 +20,5 @@ configMapGenerator:
   - ./schema/20240815053106_update-indexes.sql
   - ./schema/20240815164808_async_calls_cron_job_key.sql
   - ./schema/20240820011612_encryption_verification.sql
+  - ./schema/20240902030242_remove_notify.sql
+  - ./schema/20240903043046_enforce_runner_deployment.sql

deployment/base/kustomization.yml

@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- https://raw.githubusercontent.com/reactive-tech/kubegres/v1.18/kubegres.yaml
 - db-create
 - db-migrate
 - ftl-controller

internal/integration/actions.go

@@ -220,7 +220,13 @@ func ExpectError(action Action, expectedErrorMsg ...string) Action {
 // Deploy a module from the working directory and wait for it to become available.
 func Deploy(module string) Action {
 	return Chain(
-		Exec("ftl", "deploy", module),
+		func(t testing.TB, ic TestContext) {
+			if ic.kube {
+				Exec("ftl", "deploy", "--build-env", "GOOS=linux", "--build-env", "GOARCH=amd64", module)(t, ic)
+			} else {
+				Exec("ftl", "deploy", module)(t, ic)
+			}
+		},
 		Wait(module),
 	)
 }

internal/integration/harness.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"slices"
 	"sync"
 	"syscall"
@@ -59,6 +60,16 @@ func WithLanguages(languages ...string) Option {
 	}
 }
 
+// WithKubernetes is a Run* option that specifies tests should be run on a kube cluster
+//
+// This is only compatible with go tests
+func WithKubernetes() Option {
+	return func(o *options) {
+		o.kube = true
+		o.startController = false
+	}
+}
+
 // WithTestDataDir sets the directory from which to look for test data.
 //
 // Defaults to "testdata/<language>" if not provided.
@@ -111,6 +122,7 @@ type options struct {
 	startController bool
 	requireJava     bool
 	envars          map[string]string
+	kube            bool
 }
 
 // Run an integration test.
@@ -159,10 +171,29 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) {
 	ctx := log.ContextWithLogger(context.Background(), logger)
 	binDir := filepath.Join(rootDir, "build", "release")
 
+	if opts.kube && len(opts.languages) != 1 && opts.languages[0] != "go" {
+		t.Fatal("Kubernetes tests are only supported for golang")
+	}
+
 	buildOnce.Do(func() {
-		Infof("Building ftl")
-		err = ftlexec.Command(ctx, log.Debug, rootDir, "just", "build", "ftl").RunBuffered(ctx)
-		assert.NoError(t, err)
+		if opts.kube {
+			// This command will build a linux/amd64 version of FTL and deploy it to the kube cluster
+			Infof("Building FTL and deploying to kube")
+			err = ftlexec.Command(ctx, log.Debug, filepath.Join(rootDir, "deployment"), "just", "full-deploy").RunBuffered(ctx)
+			assert.NoError(t, err)
+			if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
+				// If we are already on linux/amd64 we don't need to rebuild, otherwise we now need a native one to interact with the kube cluster
+				Infof("Building FTL for native OS")
+				err = ftlexec.Command(ctx, log.Debug, rootDir, "just", "build", "ftl").RunBuffered(ctx)
+				assert.NoError(t, err)
+			}
+			err = ftlexec.Command(ctx, log.Debug, filepath.Join(rootDir, "deployment"), "just", "wait-for-kube").RunBuffered(ctx)
+			assert.NoError(t, err)
+		} else {
+			Infof("Building ftl")
+			err = ftlexec.Command(ctx, log.Debug, rootDir, "just", "build", "ftl").RunBuffered(ctx)
+			assert.NoError(t, err)
+		}
 		if opts.requireJava || slices.Contains(opts.languages, "java") {
 			err = ftlexec.Command(ctx, log.Debug, rootDir, "just", "build-java", "-DskipTests", "-B").RunBuffered(ctx)
 			assert.NoError(t, err)
@@ -200,6 +231,7 @@ func run(t *testing.T, actionsOrOptions ...ActionOrOption) {
 				Verbs:    verbs,
 				realT:    t,
 				language: language,
+				kube:     opts.kube,
 			}
 
 			if opts.startController {
@@ -264,6 +296,8 @@ type TestContext struct {
 	binDir string
 	// The Language under test
 	language string
+	// If the test is running on kubernetes
+	kube bool
 
 	Controller ftlv1connect.ControllerServiceClient
 	Console    pbconsoleconnect.ConsoleServiceClient