From 48ea4a6d039110fba89aa5b9c4814e16e67f0b10 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Wed, 24 Jul 2024 08:56:14 +1000 Subject: [PATCH] fix: add ability to delay runner registration (#2139) We think there may be some issues with runners registering themselves before istio is fully ready, resulting in RBAC errors. This workaround should allow for a delay that will prevent this. Possible fix for #2059 --- backend/runner/runner.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/backend/runner/runner.go b/backend/runner/runner.go index dc0754d809..a51dd8b31f 100644 --- a/backend/runner/runner.go +++ b/backend/runner/runner.go @@ -14,6 +14,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "sync" "syscall" @@ -104,8 +105,21 @@ func Start(ctx context.Context, config Config) error { } svc.state.Store(ftlv1.RunnerState_RUNNER_IDLE) - go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.RegisterRunner, svc.registrationLoop) - go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.StreamDeploymentLogs, svc.streamLogsLoop) + go func() { + // In some environments we may want a delay before registering the runner + // We have seen istio race conditions that we think this will help + startDelay := os.Getenv("FTL_RUNNER_START_DELAY") + if startDelay != "" { + delay, err := strconv.Atoi(startDelay) + if err != nil { + logger.Errorf(err, "could not parse RUNNER_START_DELAY") + } else { + time.Sleep(time.Second * time.Duration(delay)) + } + } + go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.RegisterRunner, svc.registrationLoop) + go rpc.RetryStreamingClientStream(ctx, backoff.Backoff{}, controllerClient.StreamDeploymentLogs, svc.streamLogsLoop) + }() return rpc.Serve(ctx, config.Bind, rpc.GRPC(ftlv1connect.NewVerbServiceHandler, svc),