Skip to content

Commit

Permalink
fix: suppress irrelevant log spam when scaling runners (#537)
Browse files Browse the repository at this point in the history
Also added `--recreate` flag to `ftl serve`.

Fixes #534
  • Loading branch information
alecthomas authored Nov 2, 2023
1 parent b28d9d7 commit 6584788
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 13 deletions.
3 changes: 3 additions & 0 deletions backend/common/rpc/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ func (m *metadataInterceptor) WrapStreamingHandler(req connect.StreamingHandlerF
}
err = errors.WithStack(req(ctx, s))
if err != nil {
if connect.CodeOf(err) == connect.CodeCanceled {
return nil
}
logger.Logf(m.errorLevel, "Streaming RPC failed: %s: %s", err, s.Spec().Procedure)
return err
}
Expand Down
4 changes: 3 additions & 1 deletion backend/common/rpc/rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,9 @@ func RetryStreamingClientStream[Req, Resp any](

errored = true
delay := retry.Duration()
logger.Logf(logLevel, "Stream handler failed, retrying in %s: %s", delay, err)
if !errors.Is(err, context.Canceled) {
logger.Logf(logLevel, "Stream handler failed, retrying in %s: %s", delay, err)
}
select {
case <-ctx.Done():
return
Expand Down
4 changes: 2 additions & 2 deletions backend/controller/scaling/local_scaling.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ func (l *LocalScaling) SetReplicas(ctx context.Context, replicas int, idleRunner
go func() {
logger.Infof("Starting runner: %s", config.Key)
err := runner.Start(runnerCtx, config)
if err != nil {
logger.Errorf(err, "Error starting runner: %s", err)
if err != nil && !errors.Is(err, context.Canceled) {
logger.Errorf(err, "Runner failed: %s", err)
}
}()
}
Expand Down
16 changes: 6 additions & 10 deletions cmd/ftl/cmd_serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@ import (
"github.com/TBD54566975/ftl/backend/common/exec"
"github.com/TBD54566975/ftl/backend/common/log"
"github.com/TBD54566975/ftl/backend/controller"
"github.com/TBD54566975/ftl/backend/controller/databasetesting"
"github.com/TBD54566975/ftl/backend/controller/scaling"
)

type serveCmd struct {
Bind *url.URL `help:"Starting endpoint to bind to and advertise to. Each controller and runner will increment the port by 1" default:"http://localhost:8892"`
Recreate bool `help:"Recreate the database even if it already exists." default:"false"`
Controllers int `short:"c" help:"Number of controllers to start." default:"1"`
Runners int `short:"r" help:"Number of runners to start." default:"0"`
}
Expand All @@ -29,7 +31,7 @@ const ftlContainerName = "ftl-db"
func (s *serveCmd) Run(ctx context.Context) error {
logger := log.FromContext(ctx)

dsn, err := setupDB(ctx)
dsn, err := s.setupDB(ctx)
if err != nil {
return errors.WithStack(err)
}
Expand Down Expand Up @@ -82,7 +84,7 @@ func (s *serveCmd) Run(ctx context.Context) error {
return nil
}

func setupDB(ctx context.Context) (string, error) {
func (s *serveCmd) setupDB(ctx context.Context) (string, error) {
logger := log.FromContext(ctx)
logger.Infof("Checking for FTL database")

Expand All @@ -93,7 +95,7 @@ func setupDB(ctx context.Context) (string, error) {
return "", errors.WithStack(err)
}

recreate := false
recreate := s.Recreate

if len(output) == 0 {
logger.Infof("Creating FTL database")
Expand Down Expand Up @@ -132,14 +134,8 @@ func setupDB(ctx context.Context) (string, error) {
}

dsn := fmt.Sprintf("postgres://postgres:secret@localhost:%s/%s?sslmode=disable", strings.TrimSpace(string(port)), ftlContainerName)
dsnFlag := fmt.Sprintf("--dsn=%s", dsn)

if recreate {
logger.Infof("Initializing FTL schema")
err = exec.Command(ctx, logger.GetLevel(), ".", "ftl-initdb", "--recreate", dsnFlag).Run()
} else {
err = exec.Command(ctx, logger.GetLevel(), ".", "ftl-initdb", dsnFlag).Run()
}
_, err = databasetesting.CreateForDevel(ctx, dsn, recreate)
if err != nil {
return "", errors.WithStack(err)
}
Expand Down

0 comments on commit 6584788

Please sign in to comment.