From c726b29bf5dc1990ced5f229ffc97eed2e362ff3 Mon Sep 17 00:00:00 2001 From: Stuart Douglas Date: Thu, 3 Oct 2024 15:52:04 +1000 Subject: [PATCH] fix: don't retry indefinitly with local scaling (#2974) If the runner keeps crashing dont keep restarting it --- backend/controller/scaling/localscaling/local_scaling.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/controller/scaling/localscaling/local_scaling.go b/backend/controller/scaling/localscaling/local_scaling.go index 6eccb9dbcf..fccea5e2c5 100644 --- a/backend/controller/scaling/localscaling/local_scaling.go +++ b/backend/controller/scaling/localscaling/local_scaling.go @@ -25,6 +25,8 @@ import ( var _ scaling.RunnerScaling = &localScaling{} +const maxExits = 10 + type localScaling struct { lock sync.Mutex cacheDir string @@ -71,6 +73,7 @@ type deploymentInfo struct { replicas int32 key string language string + exits int } type runnerInfo struct { cancelFunc context.CancelFunc @@ -133,7 +136,7 @@ func (l *localScaling) handleSchemaChange(ctx context.Context, msg *ftlv1.PullSc func (l *localScaling) reconcileRunners(ctx context.Context, deploymentRunners *deploymentInfo) error { // Must be called under lock logger := log.FromContext(ctx) - if deploymentRunners.replicas > 0 && !deploymentRunners.runner.Ok() { + if deploymentRunners.replicas > 0 && !deploymentRunners.runner.Ok() && deploymentRunners.exits < maxExits { if err := l.startRunner(ctx, deploymentRunners.key, deploymentRunners); err != nil { logger.Errorf(err, "Failed to start runner") return err @@ -205,6 +208,10 @@ func (l *localScaling) startRunner(ctx context.Context, deploymentKey string, in } l.lock.Lock() defer l.lock.Unlock() + info.exits++ + if info.exits >= maxExits { + logger.Errorf(fmt.Errorf("too many restarts"), "Runner failed too many times, not restarting") + } info.runner = optional.None[runnerInfo]() if l.debugPorts[info.module] == debug { delete(l.debugPorts, info.module)