fix: add ability to delay runner registration

We think there may be some issues with runners registering themselves before istio is fully ready, resulting in RBAC errors.

This workaround should allow for a delay that will prevent this.

Possible fix for #2059

backend/runner/runner.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -104,8 +105,21 @@ func Start(ctx context.Context, config Config) error {
 	}
 	svc.state.Store(ftlv1.RunnerState_RUNNER_IDLE)
 
-	go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.RegisterRunner, svc.registrationLoop)
-	go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.StreamDeploymentLogs, svc.streamLogsLoop)
+	go func() {
+		// In some environments we may want a delay before registering the runner
+		// We have seen istio race conditions that we think this will help
+		startDelay := os.Getenv("FTL_RUNNER_START_DELAY")
+		if startDelay != "" {
+			delay, err := strconv.Atoi(startDelay)
+			if err != nil {
+				logger.Errorf(err, "could not parse RUNNER_START_DELAY")
+			} else {
+				time.Sleep(time.Second * time.Duration(delay))
+			}
+		}
+		go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.RegisterRunner, svc.registrationLoop)
+		go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.StreamDeploymentLogs, svc.streamLogsLoop)
+	}()
 
 	return rpc.Serve(ctx, config.Bind,
 		rpc.GRPC(ftlv1connect.NewVerbServiceHandler, svc),