feat(go-runtime): use new configuration/secret system (#1013)

This allows secrets/config to be dynamically updated in most cases (the
envar provider being a notable exception).

```
🐚 ~/dev/ftl $ ftl config set echo.default --inline "anonymous"
🐚 ~/dev/ftl $ ftl config get echo.default
"anonymous"
🐚 ~/dev/ftl $ ftl call echo.echo       aat/configuration-go-runtime
{"message":"Hello, anonymous!!! It is 2024-03-03 07:45:21.237088 +1000 AEST!"}
🐚 ~/dev/ftl $ ftl config set echo.default --inline "Anne"
🐚 ~/dev/ftl $ ftl config get echo.default
"Anne"
🐚 ~/dev/ftl $ ftl call echo.echo
{"message":"Hello, Anne!!! It is 2024-03-03 07:44:52.2176 +1000 AEST!"}
```

I also refactored the `configuration` package such that managers and
providers are tied to their role by a type parameter.

Finally, I made a few tweaks to allow FTL to largely work offline:

- Propagate `replace` directives from Go modules into the generated main
and external-module `go.mod` files.
- Add `--[no-]console` flag that allows building of the console to be
skipped.
- TODO: Figure out how put maven builds into offline mode

Bitfile

@@ -111,6 +111,7 @@ kotlin-runtime/external-module-template.zip: kotlin-runtime/external-module-temp
   -clean
 
 %{KT_RUNTIME_OUT}: %{KT_RUNTIME_IN} %{PROTO_IN}
+  # TODO: Figure out how to make Maven build completely offline. Bizarrely "-o" does not do this.
   build:
     mvn -B -N install
     mvn -B -pl :ftl-runtime install

backend/controller/controller.go

@@ -51,6 +51,7 @@ import (
 
 type Config struct {
 	Bind                         *url.URL            `help:"Socket to bind to." default:"http://localhost:8892" env:"FTL_CONTROLLER_BIND"`
+	NoConsole                    bool                `help:"Disable the console."`
 	Advertise                    *url.URL            `help:"Endpoint the Controller should advertise (must be unique across the cluster, defaults to --bind if omitted)." env:"FTL_CONTROLLER_ADVERTISE"`
 	ConsoleURL                   *url.URL            `help:"The public URL of the console (for CORS)." env:"FTL_CONTROLLER_CONSOLE_URL"`
 	AllowOrigins                 []*url.URL          `help:"Allow CORS requests to ingress endpoints from these origins." env:"FTL_CONTROLLER_ALLOW_ORIGIN"`
@@ -78,9 +79,18 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali
 	logger := log.FromContext(ctx)
 	logger.Debugf("Starting FTL controller")
 
-	c, err := frontend.Server(ctx, config.ContentTime, config.ConsoleURL)
-	if err != nil {
-		return err
+	var consoleHandler http.Handler
+	var err error
+	if config.NoConsole {
+		consoleHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotImplemented)
+			_, _ = w.Write([]byte("Console not installed."))
+		})
+	} else {
+		consoleHandler, err = frontend.Server(ctx, config.ContentTime, config.ConsoleURL)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Bring up the DB connection and DAL.
@@ -111,7 +121,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali
 		rpc.GRPC(ftlv1connect.NewControllerServiceHandler, svc),
 		rpc.GRPC(pbconsoleconnect.NewConsoleServiceHandler, console),
 		rpc.HTTP("/ingress/", ingressHandler),
-		rpc.HTTP("/", c),
+		rpc.HTTP("/", consoleHandler),
 	)
 }
 
@@ -845,7 +855,6 @@ func (s *Service) reconcileDeployments(ctx context.Context) (time.Duration, erro
 	}
 	wg, ctx := concurrency.New(ctx, concurrency.WithConcurrencyLimit(4))
 	for _, reconcile := range reconciliation {
-		reconcile := reconcile
 		deploymentLogger := s.getDeploymentLogger(ctx, reconcile.Deployment)
 		deploymentLogger.Debugf("Reconciling %s", reconcile.Deployment)
 		deployment := model.Deployment{
@@ -860,7 +869,10 @@ func (s *Service) reconcileDeployments(ctx context.Context) (time.Duration, erro
 				if err := s.deploy(ctx, deployment); err != nil {
 					deploymentLogger.Debugf("Failed to increase deployment replicas: %s", err)
 				} else {
-					deploymentLogger.Debugf("Reconciled %s", reconcile.Deployment)
+					deploymentLogger.Debugf("Reconciled %s to %d/%d replicas", reconcile.Deployment, reconcile.AssignedReplicas+1, reconcile.RequiredReplicas)
+					if reconcile.AssignedReplicas+1 == reconcile.RequiredReplicas {
+						deploymentLogger.Infof("Deployed %s", reconcile.Deployment)
+					}
 				}
 				return nil
 			})
@@ -871,7 +883,10 @@ func (s *Service) reconcileDeployments(ctx context.Context) (time.Duration, erro
 				if err != nil {
 					deploymentLogger.Warnf("Failed to terminate runner: %s", err)
 				} else if ok {
-					deploymentLogger.Debugf("Reconciled %s", reconcile.Deployment)
+					deploymentLogger.Debugf("Reconciled %s to %d/%d replicas", reconcile.Deployment, reconcile.AssignedReplicas-1, reconcile.RequiredReplicas)
+					if reconcile.AssignedReplicas-1 == reconcile.RequiredReplicas {
+						deploymentLogger.Infof("Stopped %s", reconcile.Deployment)
+					}
 				} else {
 					deploymentLogger.Warnf("Failed to terminate runner: no runners found")
 				}

backend/controller/scaling/localscaling/devel.go

@@ -16,8 +16,8 @@ var templateDirOnce sync.Once
 
 func templateDir(ctx context.Context) string {
 	templateDirOnce.Do(func() {
-		cmd := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar")
-		err := cmd.Run()
+		// TODO: Figure out how to make maven build offline
+		err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx)
 		if err != nil {
 			panic(err)
 		}

backend/runner/runner.go

@@ -27,6 +27,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/plugin"
+	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/download"
 	"github.com/TBD54566975/ftl/internal/log"
 	"github.com/TBD54566975/ftl/internal/model"
@@ -218,6 +219,7 @@ func (s *Service) Deploy(ctx context.Context, req *connect.Request[ftlv1.DeployR
 		ftlv1connect.NewVerbServiceClient,
 		plugin.WithEnvars(
 			"FTL_ENDPOINT="+s.config.ControllerEndpoint.String(),
+			"FTL_CONFIG="+filepath.Join(internal.GitRoot(""), "ftl-project.toml"),
 			"FTL_OBSERVABILITY_ENDPOINT="+s.config.ControllerEndpoint.String(),
 		),
 	)

cmd/ftl/cmd_config.go

@@ -7,35 +7,18 @@ import (
 	"io"
 	"os"
 
-	"github.com/alecthomas/kong"
-
 	"github.com/TBD54566975/ftl/common/configuration"
 )
 
-type mutableConfigProviderMixin struct {
-	configuration.InlineProvider
-	configuration.EnvarProvider[configuration.EnvarTypeConfig]
-}
-
-func (s *mutableConfigProviderMixin) newConfigManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) {
-	return configuration.New(ctx, resolver, []configuration.Provider{s.InlineProvider, s.EnvarProvider})
-}
-
 type configCmd struct {
-	configuration.ProjectConfigResolver[configuration.FromConfig]
+	configuration.DefaultConfigMixin
 
 	List  configListCmd  `cmd:"" help:"List configuration."`
 	Get   configGetCmd   `cmd:"" help:"Get a configuration value."`
 	Set   configSetCmd   `cmd:"" help:"Set a configuration value."`
 	Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."`
 }
 
-func (s *configCmd) newConfigManager(ctx context.Context) (*configuration.Manager, error) {
-	mp := mutableConfigProviderMixin{}
-	_ = kong.ApplyDefaults(&mp)
-	return mp.newConfigManager(ctx, s.ProjectConfigResolver)
-}
-
 func (s *configCmd) Help() string {
 	return `
 Configuration values are used to store non-sensitive information such as URLs,
@@ -49,7 +32,7 @@ type configListCmd struct {
 }
 
 func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := scmd.newConfigManager(ctx)
+	sm, err := scmd.NewConfigurationManager(ctx)
 	if err != nil {
 		return err
 	}
@@ -95,7 +78,7 @@ Returns a JSON-encoded configuration value.
 }
 
 func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := scmd.newConfigManager(ctx)
+	sm, err := scmd.NewConfigurationManager(ctx)
 	if err != nil {
 		return err
 	}
@@ -115,15 +98,13 @@ func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error {
 }
 
 type configSetCmd struct {
-	mutableConfigProviderMixin
-
 	JSON  bool              `help:"Assume input value is JSON."`
 	Ref   configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 	Value *string           `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""`
 }
 
 func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver)
+	sm, err := scmd.NewConfigurationManager(ctx)
 	if err != nil {
 		return err
 	}
@@ -154,13 +135,11 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error {
 }
 
 type configUnsetCmd struct {
-	mutableConfigProviderMixin
-
 	Ref configuration.Ref `arg:"" help:"Configuration reference in the form [<module>.]<name>."`
 }
 
 func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd) error {
-	sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver)
+	sm, err := scmd.NewConfigurationManager(ctx)
 	if err != nil {
 		return err
 	}

cmd/ftl/cmd_secret.go

@@ -7,41 +7,21 @@ import (
 	"io"
 	"os"
 
-	"github.com/alecthomas/kong"
 	"github.com/mattn/go-isatty"
 	"golang.org/x/term"
 
 	"github.com/TBD54566975/ftl/common/configuration"
 )
 
-type mutableSecretProviderMixin struct {
-	configuration.InlineProvider
-	configuration.KeychainProvider
-	configuration.EnvarProvider[configuration.EnvarTypeSecrets]
-	configuration.OnePasswordProvider
-}
-
-func (s *mutableSecretProviderMixin) newSecretsManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) {
-	return configuration.New(ctx, resolver, []configuration.Provider{
-		s.InlineProvider, s.KeychainProvider, s.EnvarProvider, s.OnePasswordProvider,
-	})
-}
-
 type secretCmd struct {
-	configuration.ProjectConfigResolver[configuration.FromSecrets]
+	configuration.DefaultSecretsMixin
 
 	List  secretListCmd  `cmd:"" help:"List secrets."`
 	Get   secretGetCmd   `cmd:"" help:"Get a secret."`
 	Set   secretSetCmd   `cmd:"" help:"Set a secret."`
 	Unset secretUnsetCmd `cmd:"" help:"Unset a secret."`
 }
 
-func (s *secretCmd) newSecretsManager(ctx context.Context) (*configuration.Manager, error) {
-	mp := mutableSecretProviderMixin{}
-	_ = kong.ApplyDefaults(&mp)
-	return mp.newSecretsManager(ctx, s.ProjectConfigResolver)
-}
-
 func (s *secretCmd) Help() string {
 	return `
 Secrets are used to store sensitive information such as passwords, tokens, and
@@ -58,7 +38,7 @@ type secretListCmd struct {
 }
 
 func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := scmd.newSecretsManager(ctx)
+	sm, err := scmd.NewSecretsManager(ctx)
 	if err != nil {
 		return err
 	}
@@ -104,7 +84,7 @@ Returns a JSON-encoded secret value.
 }
 
 func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := scmd.newSecretsManager(ctx)
+	sm, err := scmd.NewSecretsManager(ctx)
 	if err != nil {
 		return err
 	}
@@ -124,14 +104,12 @@ func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error {
 }
 
 type secretSetCmd struct {
-	mutableSecretProviderMixin
-
 	JSON bool              `help:"Assume input value is JSON."`
 	Ref  configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
 func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver)
+	sm, err := scmd.NewSecretsManager(ctx)
 	if err != nil {
 		return err
 	}
@@ -168,13 +146,11 @@ func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error {
 }
 
 type secretUnsetCmd struct {
-	mutableSecretProviderMixin
-
 	Ref configuration.Ref `arg:"" help:"Secret reference in the form [<module>.]<name>."`
 }
 
 func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd) error {
-	sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver)
+	sm, err := scmd.NewSecretsManager(ctx)
 	if err != nil {
 		return err
 	}

cmd/ftl/cmd_serve.go

@@ -32,6 +32,7 @@ import (
 type serveCmd struct {
 	Bind           *url.URL      `help:"Starting endpoint to bind to and advertise to. Each controller and runner will increment the port by 1" default:"http://localhost:8892"`
 	AllowOrigins   []*url.URL    `help:"Allow CORS requests to ingress endpoints from these origins." env:"FTL_CONTROLLER_ALLOW_ORIGIN"`
+	NoConsole      bool          `help:"Disable the console."`
 	DBPort         int           `help:"Port to use for the database." default:"54320"`
 	Recreate       bool          `help:"Recreate the database even if it already exists." default:"false"`
 	Controllers    int           `short:"c" help:"Number of controllers to start." default:"1"`
@@ -98,6 +99,7 @@ func (s *serveCmd) Run(ctx context.Context) error {
 			Bind:         controllerAddresses[i],
 			DSN:          dsn,
 			AllowOrigins: s.AllowOrigins,
+			NoConsole:    s.NoConsole,
 		}
 		if err := kong.ApplyDefaults(&config); err != nil {
 			return err

common/configuration/1password.go

@@ -17,9 +17,9 @@ type OnePasswordProvider struct {
 	OnePassword bool `name:"op" help:"Write 1Password secret references - does not write to 1Password." group:"Provider:" xor:"configwriter"`
 }
 
-var _ MutableProvider = OnePasswordProvider{}
+var _ MutableProvider[Secrets] = OnePasswordProvider{}
 
-func (o OnePasswordProvider) Key() string                               { return "op" }
+func (o OnePasswordProvider) Key() Secrets                              { return "op" }
 func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil }
 
 func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) {

common/configuration/api.go

@@ -1,4 +1,17 @@
-// Package configuration is a generic configuration and secret management API.
+// Package configuration is the FTL configuration and secret management API.
+//
+// The full design is documented [here].
+//
+// A [Manager] is the high-level interface to storing, listing, and retrieving
+// secrets and configuration. A [Resolver] is the next layer, mapping
+// names to a storage location key such as environment variables, keychain, etc.
+// The [Provider] is the final layer, responsible for actually storing and
+// retrieving values in concrete storage.
+//
+// A constructed [Manager] and its providers are parametric on either secrets or
+// configuration and thus cannot be used interchangeably.
+//
+// [here]: https://hackmd.io/@ftl/S1e6YVEuq6
 package configuration
 
 import (
@@ -70,14 +83,14 @@ type Resolver interface {
 }
 
 // Provider is a generic interface for storing and retrieving configuration and secrets.
-type Provider interface {
-	Key() string
+type Provider[R Role] interface {
+	Key() R
 	Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error)
 }
 
 // A MutableProvider is a Provider that can update configuration.
-type MutableProvider interface {
-	Provider
+type MutableProvider[R Role] interface {
+	Provider[R]
 	// Writer returns true if this provider should be used to store configuration.
 	//
 	// Only one provider should return true.

common/configuration/context.go

@@ -0,0 +1,37 @@
+package configuration
+
+import "context"
+
+type contextKeySecrets struct{}
+
+type contextKeyConfig struct{}
+
+// ContextWithSecrets adds a secrets manager to the given context.
+func ContextWithSecrets(ctx context.Context, secretsManager *Manager[Secrets]) context.Context {
+	return context.WithValue(ctx, contextKeySecrets{}, secretsManager)
+}
+
+// SecretsFromContext retrieves the secrets configuration.Manager previously
+// added to the context with [ContextWithConfig].
+func SecretsFromContext(ctx context.Context) *Manager[Secrets] {
+	s, ok := ctx.Value(contextKeySecrets{}).(*Manager[Secrets])
+	if !ok {
+		panic("no secrets manager in context")
+	}
+	return s
+}
+
+// ContextWithConfig adds a configuration manager to the given context.
+func ContextWithConfig(ctx context.Context, configManager *Manager[Configuration]) context.Context {
+	return context.WithValue(ctx, contextKeyConfig{}, configManager)
+}
+
+// ConfigFromContext retrieves the configuration.Manager previously added to the
+// context with [ContextWithConfig].
+func ConfigFromContext(ctx context.Context) *Manager[Configuration] {
+	m, ok := ctx.Value(contextKeyConfig{}).(*Manager[Configuration])
+	if !ok {
+		panic("no configuration manager in context")
+	}
+	return m
+}