From 56dbfad0b15d8903301fb6122ab9cdea67d422c1 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Sun, 3 Mar 2024 07:44:02 +1000 Subject: [PATCH] feat(go-runtime): use new configuration/secret system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows secrets/config to be dynamically updated in most cases (the envar provider being a notable exception). ``` 🐚 ~/dev/ftl $ ftl config set echo.default --inline "anonymous" 🐚 ~/dev/ftl $ ftl config get echo.default "anonymous" 🐚 ~/dev/ftl $ ftl call echo.echo aat/configuration-go-runtime {"message":"Hello, anonymous!!! It is 2024-03-03 07:45:21.237088 +1000 AEST!"} 🐚 ~/dev/ftl $ ftl config set echo.default --inline "Anne" 🐚 ~/dev/ftl $ ftl config get echo.default "Anne" 🐚 ~/dev/ftl $ ftl call echo.echo {"message":"Hello, Anne!!! It is 2024-03-03 07:44:52.2176 +1000 AEST!"} ``` Also made a few tweaks to allow FTL to largely work offline: - Propagate `replace` directives from Go modules into the generated main and external-module `go.mod` files. - Add `--[no-]console` flag that allows building of the console to be skipped. - Put Bit maven builds into offline mode (does not work fully, I think there's another flag to force Maven to use the local repository that needs to be used, but I'm offline and there's nothing in `--help`). --- Bitfile | 1 + backend/controller/controller.go | 23 ++-- .../controller/scaling/localscaling/devel.go | 4 +- backend/runner/runner.go | 2 + cmd/ftl/cmd_config.go | 31 +---- cmd/ftl/cmd_secret.go | 34 +----- cmd/ftl/cmd_serve.go | 2 + common/configuration/1password.go | 4 +- common/configuration/api.go | 23 +++- common/configuration/context.go | 37 ++++++ common/configuration/defaults.go | 63 +++++++++++ common/configuration/envar.go | 46 ++++---- common/configuration/inline.go | 14 +-- common/configuration/keychain.go | 4 +- common/configuration/manager.go | 70 ++++++------ common/configuration/manager_test.go | 69 +++++++---- common/configuration/projectconfig.go | 107 +++++++++--------- .../configuration/testdata/ftl-project.toml | 7 +- examples/go/echo/echo.go | 4 +- examples/go/echo/go.mod | 3 + examples/go/echo/go.sum | 6 + ftl-project.toml | 6 + .../_ftl.tmpl/go/main/go.mod.tmpl | 4 + go-runtime/compile/build.go | 84 +++++++++----- .../_ftl/go/modules/go.mod.tmpl | 8 ++ go-runtime/ftl/config.go | 20 ++-- go-runtime/ftl/config_test.go | 11 +- go-runtime/ftl/secrets.go | 20 ++-- go-runtime/ftl/secrets_test.go | 13 ++- go-runtime/ftl/testdata/ftl-project.toml | 8 ++ go-runtime/server/server.go | 30 +++-- internal/observability/client.go | 10 +- scripts/autofmt | 2 +- 33 files changed, 489 insertions(+), 281 deletions(-) create mode 100644 common/configuration/context.go create mode 100644 common/configuration/defaults.go create mode 100644 ftl-project.toml create mode 100644 go-runtime/ftl/testdata/ftl-project.toml diff --git a/Bitfile b/Bitfile index 18a13f3717..286787c00f 100644 --- a/Bitfile +++ b/Bitfile @@ -111,6 +111,7 @@ kotlin-runtime/external-module-template.zip: kotlin-runtime/external-module-temp -clean %{KT_RUNTIME_OUT}: %{KT_RUNTIME_IN} %{PROTO_IN} + # TODO: Figure out how to make Maven build completely offline. Bizarrely "-o" does not do this. build: mvn -B -N install mvn -B -pl :ftl-runtime install diff --git a/backend/controller/controller.go b/backend/controller/controller.go index 3310d932f4..473fbd6160 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -51,6 +51,7 @@ import ( type Config struct { Bind *url.URL `help:"Socket to bind to." default:"http://localhost:8892" env:"FTL_CONTROLLER_BIND"` + NoConsole bool `help:"Disable the console."` Advertise *url.URL `help:"Endpoint the Controller should advertise (must be unique across the cluster, defaults to --bind if omitted)." env:"FTL_CONTROLLER_ADVERTISE"` ConsoleURL *url.URL `help:"The public URL of the console (for CORS)." env:"FTL_CONTROLLER_CONSOLE_URL"` AllowOrigins []*url.URL `help:"Allow CORS requests to ingress endpoints from these origins." env:"FTL_CONTROLLER_ALLOW_ORIGIN"` @@ -78,9 +79,18 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali logger := log.FromContext(ctx) logger.Debugf("Starting FTL controller") - c, err := frontend.Server(ctx, config.ContentTime, config.ConsoleURL) - if err != nil { - return err + var consoleHandler http.Handler + var err error + if config.NoConsole { + consoleHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + _, _ = w.Write([]byte("Console not installed.")) + }) + } else { + consoleHandler, err = frontend.Server(ctx, config.ContentTime, config.ConsoleURL) + if err != nil { + return err + } } // Bring up the DB connection and DAL. @@ -111,7 +121,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali rpc.GRPC(ftlv1connect.NewControllerServiceHandler, svc), rpc.GRPC(pbconsoleconnect.NewConsoleServiceHandler, console), rpc.HTTP("/ingress/", ingressHandler), - rpc.HTTP("/", c), + rpc.HTTP("/", consoleHandler), ) } @@ -845,7 +855,6 @@ func (s *Service) reconcileDeployments(ctx context.Context) (time.Duration, erro } wg, ctx := concurrency.New(ctx, concurrency.WithConcurrencyLimit(4)) for _, reconcile := range reconciliation { - reconcile := reconcile deploymentLogger := s.getDeploymentLogger(ctx, reconcile.Deployment) deploymentLogger.Debugf("Reconciling %s", reconcile.Deployment) deployment := model.Deployment{ @@ -860,7 +869,7 @@ func (s *Service) reconcileDeployments(ctx context.Context) (time.Duration, erro if err := s.deploy(ctx, deployment); err != nil { deploymentLogger.Debugf("Failed to increase deployment replicas: %s", err) } else { - deploymentLogger.Debugf("Reconciled %s", reconcile.Deployment) + deploymentLogger.Infof("Reconciled %s to %d/%d replicas", reconcile.Deployment, reconcile.AssignedReplicas+1, reconcile.RequiredReplicas) } return nil }) @@ -871,7 +880,7 @@ func (s *Service) reconcileDeployments(ctx context.Context) (time.Duration, erro if err != nil { deploymentLogger.Warnf("Failed to terminate runner: %s", err) } else if ok { - deploymentLogger.Debugf("Reconciled %s", reconcile.Deployment) + deploymentLogger.Infof("Reconciled %s to %d/%d replicas", reconcile.Deployment, reconcile.AssignedReplicas-1, reconcile.RequiredReplicas) } else { deploymentLogger.Warnf("Failed to terminate runner: no runners found") } diff --git a/backend/controller/scaling/localscaling/devel.go b/backend/controller/scaling/localscaling/devel.go index e5a5d76dd9..5e197cdee1 100644 --- a/backend/controller/scaling/localscaling/devel.go +++ b/backend/controller/scaling/localscaling/devel.go @@ -16,8 +16,8 @@ var templateDirOnce sync.Once func templateDir(ctx context.Context) string { templateDirOnce.Do(func() { - cmd := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar") - err := cmd.Run() + // TODO: Figure out how to make maven build offline + err := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar").RunBuffered(ctx) if err != nil { panic(err) } diff --git a/backend/runner/runner.go b/backend/runner/runner.go index 3b0c4db144..9bb44d4d18 100644 --- a/backend/runner/runner.go +++ b/backend/runner/runner.go @@ -27,6 +27,7 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/common/plugin" + "github.com/TBD54566975/ftl/internal" "github.com/TBD54566975/ftl/internal/download" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" @@ -218,6 +219,7 @@ func (s *Service) Deploy(ctx context.Context, req *connect.Request[ftlv1.DeployR ftlv1connect.NewVerbServiceClient, plugin.WithEnvars( "FTL_ENDPOINT="+s.config.ControllerEndpoint.String(), + "FTL_CONFIG="+filepath.Join(internal.GitRoot(""), "ftl-project.toml"), "FTL_OBSERVABILITY_ENDPOINT="+s.config.ControllerEndpoint.String(), ), ) diff --git a/cmd/ftl/cmd_config.go b/cmd/ftl/cmd_config.go index 6708c6ac58..ecd6877c8e 100644 --- a/cmd/ftl/cmd_config.go +++ b/cmd/ftl/cmd_config.go @@ -7,22 +7,11 @@ import ( "io" "os" - "github.com/alecthomas/kong" - "github.com/TBD54566975/ftl/common/configuration" ) -type mutableConfigProviderMixin struct { - configuration.InlineProvider - configuration.EnvarProvider[configuration.EnvarTypeConfig] -} - -func (s *mutableConfigProviderMixin) newConfigManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) { - return configuration.New(ctx, resolver, []configuration.Provider{s.InlineProvider, s.EnvarProvider}) -} - type configCmd struct { - configuration.ProjectConfigResolver[configuration.FromConfig] + configuration.DefaultConfigMixin List configListCmd `cmd:"" help:"List configuration."` Get configGetCmd `cmd:"" help:"Get a configuration value."` @@ -30,12 +19,6 @@ type configCmd struct { Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."` } -func (s *configCmd) newConfigManager(ctx context.Context) (*configuration.Manager, error) { - mp := mutableConfigProviderMixin{} - _ = kong.ApplyDefaults(&mp) - return mp.newConfigManager(ctx, s.ProjectConfigResolver) -} - func (s *configCmd) Help() string { return ` Configuration values are used to store non-sensitive information such as URLs, @@ -49,7 +32,7 @@ type configListCmd struct { } func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := scmd.newConfigManager(ctx) + sm, err := scmd.NewConfigurationManager(ctx) if err != nil { return err } @@ -95,7 +78,7 @@ Returns a JSON-encoded configuration value. } func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := scmd.newConfigManager(ctx) + sm, err := scmd.NewConfigurationManager(ctx) if err != nil { return err } @@ -115,15 +98,13 @@ func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error { } type configSetCmd struct { - mutableConfigProviderMixin - JSON bool `help:"Assume input value is JSON."` Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""` } func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver) + sm, err := scmd.NewConfigurationManager(ctx) if err != nil { return err } @@ -154,13 +135,11 @@ func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error { } type configUnsetCmd struct { - mutableConfigProviderMixin - Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` } func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd) error { - sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver) + sm, err := scmd.NewConfigurationManager(ctx) if err != nil { return err } diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go index f33502e83e..5ae5db1929 100644 --- a/cmd/ftl/cmd_secret.go +++ b/cmd/ftl/cmd_secret.go @@ -7,28 +7,14 @@ import ( "io" "os" - "github.com/alecthomas/kong" "github.com/mattn/go-isatty" "golang.org/x/term" "github.com/TBD54566975/ftl/common/configuration" ) -type mutableSecretProviderMixin struct { - configuration.InlineProvider - configuration.KeychainProvider - configuration.EnvarProvider[configuration.EnvarTypeSecrets] - configuration.OnePasswordProvider -} - -func (s *mutableSecretProviderMixin) newSecretsManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) { - return configuration.New(ctx, resolver, []configuration.Provider{ - s.InlineProvider, s.KeychainProvider, s.EnvarProvider, s.OnePasswordProvider, - }) -} - type secretCmd struct { - configuration.ProjectConfigResolver[configuration.FromSecrets] + configuration.DefaultSecretsMixin List secretListCmd `cmd:"" help:"List secrets."` Get secretGetCmd `cmd:"" help:"Get a secret."` @@ -36,12 +22,6 @@ type secretCmd struct { Unset secretUnsetCmd `cmd:"" help:"Unset a secret."` } -func (s *secretCmd) newSecretsManager(ctx context.Context) (*configuration.Manager, error) { - mp := mutableSecretProviderMixin{} - _ = kong.ApplyDefaults(&mp) - return mp.newSecretsManager(ctx, s.ProjectConfigResolver) -} - func (s *secretCmd) Help() string { return ` Secrets are used to store sensitive information such as passwords, tokens, and @@ -58,7 +38,7 @@ type secretListCmd struct { } func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := scmd.newSecretsManager(ctx) + sm, err := scmd.NewSecretsManager(ctx) if err != nil { return err } @@ -104,7 +84,7 @@ Returns a JSON-encoded secret value. } func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := scmd.newSecretsManager(ctx) + sm, err := scmd.NewSecretsManager(ctx) if err != nil { return err } @@ -124,14 +104,12 @@ func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error { } type secretSetCmd struct { - mutableSecretProviderMixin - JSON bool `help:"Assume input value is JSON."` Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` } func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver) + sm, err := scmd.NewSecretsManager(ctx) if err != nil { return err } @@ -168,13 +146,11 @@ func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error { } type secretUnsetCmd struct { - mutableSecretProviderMixin - Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` } func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd) error { - sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver) + sm, err := scmd.NewSecretsManager(ctx) if err != nil { return err } diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go index 4fd0f24b6e..817efa92ff 100644 --- a/cmd/ftl/cmd_serve.go +++ b/cmd/ftl/cmd_serve.go @@ -32,6 +32,7 @@ import ( type serveCmd struct { Bind *url.URL `help:"Starting endpoint to bind to and advertise to. Each controller and runner will increment the port by 1" default:"http://localhost:8892"` AllowOrigins []*url.URL `help:"Allow CORS requests to ingress endpoints from these origins." env:"FTL_CONTROLLER_ALLOW_ORIGIN"` + NoConsole bool `help:"Disable the console."` DBPort int `help:"Port to use for the database." default:"54320"` Recreate bool `help:"Recreate the database even if it already exists." default:"false"` Controllers int `short:"c" help:"Number of controllers to start." default:"1"` @@ -98,6 +99,7 @@ func (s *serveCmd) Run(ctx context.Context) error { Bind: controllerAddresses[i], DSN: dsn, AllowOrigins: s.AllowOrigins, + NoConsole: s.NoConsole, } if err := kong.ApplyDefaults(&config); err != nil { return err diff --git a/common/configuration/1password.go b/common/configuration/1password.go index 444f294b82..3361d31853 100644 --- a/common/configuration/1password.go +++ b/common/configuration/1password.go @@ -17,9 +17,9 @@ type OnePasswordProvider struct { OnePassword bool `name:"op" help:"Write 1Password secret references - does not write to 1Password." group:"Provider:" xor:"configwriter"` } -var _ MutableProvider = OnePasswordProvider{} +var _ MutableProvider[Secrets] = OnePasswordProvider{} -func (o OnePasswordProvider) Key() string { return "op" } +func (o OnePasswordProvider) Key() Secrets { return "op" } func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { diff --git a/common/configuration/api.go b/common/configuration/api.go index c6110e1de4..0b82ea49d5 100644 --- a/common/configuration/api.go +++ b/common/configuration/api.go @@ -1,4 +1,17 @@ -// Package configuration is a generic configuration and secret management API. +// Package configuration is the FTL configuration and secret management API. +// +// The full design is documented [here]. +// +// A [Manager] is the high-level interface to storing, listing, and retrieving +// secrets and configuration. A [Resolver] is the next layer, mapping +// names to a storage location key such as environment variables, keychain, etc. +// The [Provider] is the final layer, responsible for actually storing and +// retrieving values in concrete storage. +// +// A constructed [Manager] and its providers are parametric on either secrets or +// configuration and thus cannot be used interchangeably. +// +// [here]: https://hackmd.io/@ftl/S1e6YVEuq6 package configuration import ( @@ -70,14 +83,14 @@ type Resolver interface { } // Provider is a generic interface for storing and retrieving configuration and secrets. -type Provider interface { - Key() string +type Provider[K Kind] interface { + Key() K Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) } // A MutableProvider is a Provider that can update configuration. -type MutableProvider interface { - Provider +type MutableProvider[K Kind] interface { + Provider[K] // Writer returns true if this provider should be used to store configuration. // // Only one provider should return true. diff --git a/common/configuration/context.go b/common/configuration/context.go new file mode 100644 index 0000000000..049a688e70 --- /dev/null +++ b/common/configuration/context.go @@ -0,0 +1,37 @@ +package configuration + +import "context" + +type contextKeySecrets struct{} + +type contextKeyConfig struct{} + +// ContextWithSecrets adds a secrets manager to the given context. +func ContextWithSecrets(ctx context.Context, secretsManager *Manager[Secrets]) context.Context { + return context.WithValue(ctx, contextKeySecrets{}, secretsManager) +} + +// SecretsFromContext retrieves the secrets configuration.Manager previously +// added to the context with [ContextWithConfig]. +func SecretsFromContext(ctx context.Context) *Manager[Secrets] { + s, ok := ctx.Value(contextKeySecrets{}).(*Manager[Secrets]) + if !ok { + panic("no secrets manager in context") + } + return s +} + +// ContextWithConfig adds a configuration manager to the given context. +func ContextWithConfig(ctx context.Context, configManager *Manager[Configuration]) context.Context { + return context.WithValue(ctx, contextKeyConfig{}, configManager) +} + +// ConfigFromContext retrieves the configuration.Manager previously added to the +// context with [ContextWithConfig]. +func ConfigFromContext(ctx context.Context) *Manager[Configuration] { + m, ok := ctx.Value(contextKeyConfig{}).(*Manager[Configuration]) + if !ok { + panic("no configuration manager in context") + } + return m +} diff --git a/common/configuration/defaults.go b/common/configuration/defaults.go new file mode 100644 index 0000000000..e2454481b2 --- /dev/null +++ b/common/configuration/defaults.go @@ -0,0 +1,63 @@ +package configuration + +import ( + "context" + + "github.com/alecthomas/kong" +) + +// NewConfigurationManager constructs a new [Manager] with the default providers for configuration. +func NewConfigurationManager(ctx context.Context, configPath string) (*Manager[Configuration], error) { + conf := DefaultConfigMixin{ + ProjectConfigResolver: ProjectConfigResolver[Configuration]{ + Config: configPath, + }, + } + _ = kong.ApplyDefaults(&conf) + return conf.NewConfigurationManager(ctx) +} + +// DefaultConfigMixin is a Kong mixin that provides the default configuration manager. +type DefaultConfigMixin struct { + ProjectConfigResolver[Configuration] + InlineProvider[Configuration] + EnvarProvider[Configuration] +} + +// NewConfigurationManager creates a new configuration manager with the default configuration providers. +func (d DefaultConfigMixin) NewConfigurationManager(ctx context.Context) (*Manager[Configuration], error) { + return New(ctx, &d.ProjectConfigResolver, []Provider[Configuration]{ + d.InlineProvider, + d.EnvarProvider, + }) +} + +// NewSecretsManager constructs a new [Manager] with the default providers for secrets. +func NewSecretsManager(ctx context.Context, configPath string) (*Manager[Secrets], error) { + conf := DefaultSecretsMixin{ + ProjectConfigResolver: ProjectConfigResolver[Secrets]{ + Config: configPath, + }, + } + _ = kong.ApplyDefaults(&conf) + return conf.NewSecretsManager(ctx) +} + +// DefaultSecretsMixin is a Kong mixin that provides the default secrets manager. +type DefaultSecretsMixin struct { + ProjectConfigResolver[Secrets] + InlineProvider[Secrets] + EnvarProvider[Secrets] + KeychainProvider + OnePasswordProvider +} + +// NewSecretsManager creates a new secrets manager with the default secret providers. +func (d DefaultSecretsMixin) NewSecretsManager(ctx context.Context) (*Manager[Secrets], error) { + return New(ctx, &d.ProjectConfigResolver, []Provider[Secrets]{ + d.InlineProvider, + d.EnvarProvider, + d.KeychainProvider, + d.OnePasswordProvider, + }) +} diff --git a/common/configuration/envar.go b/common/configuration/envar.go index fd25be630e..efc9bf6808 100644 --- a/common/configuration/envar.go +++ b/common/configuration/envar.go @@ -8,26 +8,17 @@ import ( "os" ) -type EnvarType interface{ prefix() string } - -type EnvarTypeConfig struct{} - -func (EnvarTypeConfig) prefix() string { return "FTL_CONFIG_" } - -type EnvarTypeSecrets struct{} - -func (EnvarTypeSecrets) prefix() string { return "FTL_SECRET_" } - -// EnvarProvider is a configuration provider that reads from environment variables. -type EnvarProvider[T EnvarType] struct { +// EnvarProvider is a configuration provider that reads secrets or configuration +// from environment variables. +type EnvarProvider[K Kind] struct { Envar bool `help:"Print configuration as environment variables." xor:"configwriter" group:"Provider:"` } -var _ MutableProvider = EnvarProvider[EnvarTypeConfig]{} +var _ MutableProvider[Configuration] = EnvarProvider[Configuration]{} -func (EnvarProvider[T]) Key() string { return "envar" } +func (EnvarProvider[K]) Key() K { return "envar" } -func (e EnvarProvider[T]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { +func (e EnvarProvider[K]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { // FTL__[]_ where and are base64 encoded. envar := e.key(ref) @@ -38,19 +29,19 @@ func (e EnvarProvider[T]) Load(ctx context.Context, ref Ref, key *url.URL) ([]by return nil, fmt.Errorf("environment variable %q is not set: %w", envar, ErrNotFound) } -func (e EnvarProvider[T]) Delete(ctx context.Context, ref Ref) error { +func (e EnvarProvider[K]) Delete(ctx context.Context, ref Ref) error { return nil } -func (e EnvarProvider[T]) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { +func (e EnvarProvider[K]) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { envar := e.key(ref) fmt.Printf("%s=%s\n", envar, base64.RawStdEncoding.EncodeToString(value)) return &url.URL{Scheme: "envar", Host: ref.Name}, nil } -func (e EnvarProvider[T]) Writer() bool { return e.Envar } +func (e EnvarProvider[K]) Writer() bool { return e.Envar } -func (e EnvarProvider[T]) key(ref Ref) string { +func (e EnvarProvider[K]) key(ref Ref) string { key := e.prefix() if m, ok := ref.Module.Get(); ok { key += base64.RawStdEncoding.EncodeToString([]byte(m)) + "_" @@ -59,15 +50,24 @@ func (e EnvarProvider[T]) key(ref Ref) string { return key } -func (EnvarProvider[T]) prefix() string { - var t T - return t.prefix() +func (EnvarProvider[K]) prefix() string { + var k K + switch any(k).(type) { + case Configuration: + return "FTL_CONFIG_" + + case Secrets: + return "FTL_SECRET_" + + default: + panic(fmt.Sprintf("unexpected configuration kind %T", k)) + } } // I don't think there's a need to parse environment variables, but let's keep // this around for a bit just in case, as it was a PITA to write. // -// func (e EnvarProvider[T]) entryForEnvar(env string) (Entry, error) { +// func (e EnvarProvider[K]) entryForEnvar(env string) (Entry, error) { // parts := strings.SplitN(env, "=", 2) // if !strings.HasPrefix(parts[0], e.prefix()) { // return Entry{}, fmt.Errorf("invalid environment variable %q", parts[0]) diff --git a/common/configuration/inline.go b/common/configuration/inline.go index 887d6f0a1a..edd7fe1402 100644 --- a/common/configuration/inline.go +++ b/common/configuration/inline.go @@ -8,17 +8,17 @@ import ( ) // InlineProvider is a configuration provider that stores configuration in its key. -type InlineProvider struct { +type InlineProvider[K Kind] struct { Inline bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"configwriter"` } -var _ MutableProvider = InlineProvider{} +var _ MutableProvider[Configuration] = InlineProvider[Configuration]{} -func (InlineProvider) Key() string { return "inline" } +func (InlineProvider[K]) Key() K { return "inline" } -func (i InlineProvider) Writer() bool { return i.Inline } +func (i InlineProvider[K]) Writer() bool { return i.Inline } -func (InlineProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { +func (InlineProvider[K]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { data, err := base64.RawStdEncoding.DecodeString(key.Host) if err != nil { return nil, fmt.Errorf("invalid base64 data in inline configuration: %w", err) @@ -26,11 +26,11 @@ func (InlineProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, return data, nil } -func (InlineProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { +func (InlineProvider[K]) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { b64 := base64.RawStdEncoding.EncodeToString(value) return &url.URL{Scheme: "inline", Host: b64}, nil } -func (InlineProvider) Delete(ctx context.Context, ref Ref) error { +func (InlineProvider[K]) Delete(ctx context.Context, ref Ref) error { return nil } diff --git a/common/configuration/keychain.go b/common/configuration/keychain.go index 4a6eb66221..5fe100e662 100644 --- a/common/configuration/keychain.go +++ b/common/configuration/keychain.go @@ -14,9 +14,9 @@ type KeychainProvider struct { Keychain bool `help:"Write to the system keychain." group:"Provider:" xor:"configwriter"` } -var _ MutableProvider = KeychainProvider{} +var _ MutableProvider[Secrets] = KeychainProvider{} -func (k KeychainProvider) Key() string { return "keychain" } +func (k KeychainProvider) Key() Secrets { return "keychain" } func (k KeychainProvider) Writer() bool { return k.Keychain } diff --git a/common/configuration/manager.go b/common/configuration/manager.go index ec89d48566..5afb331f11 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -6,24 +6,35 @@ import ( "errors" "fmt" "strings" + + "github.com/alecthomas/types/optional" ) +// Kind of [Manager], either Secrets or Configuration. +type Kind interface { + Secrets | Configuration +} + +type Secrets string + +type Configuration string + // Manager is a high-level configuration manager that abstracts the details of // the Resolver and Provider interfaces. -type Manager struct { - providers map[string]Provider - writer MutableProvider +type Manager[K Kind] struct { + providers map[K]Provider[K] + writer MutableProvider[K] resolver Resolver } // New configuration manager. -func New(ctx context.Context, resolver Resolver, providers []Provider) (*Manager, error) { - m := &Manager{ - providers: map[string]Provider{}, +func New[K Kind](ctx context.Context, resolver Resolver, providers []Provider[K]) (*Manager[K], error) { + m := &Manager[K]{ + providers: map[K]Provider[K]{}, } for _, p := range providers { m.providers[p.Key()] = p - if mutable, ok := p.(MutableProvider); ok && mutable.Writer() { + if mutable, ok := p.(MutableProvider[K]); ok && mutable.Writer() { if m.writer != nil { return nil, fmt.Errorf("multiple writers %s and %s", m.writer.Key(), p.Key()) } @@ -36,14 +47,14 @@ func New(ctx context.Context, resolver Resolver, providers []Provider) (*Manager // Mutable returns an error if the configuration manager doesn't have a // writeable provider configured. -func (m *Manager) Mutable() error { +func (m *Manager[K]) Mutable() error { if m.writer != nil { return nil } writers := []string{} for _, p := range m.providers { - if mutable, ok := p.(MutableProvider); ok { - writers = append(writers, "--"+mutable.Key()) + if mutable, ok := p.(MutableProvider[K]); ok { + writers = append(writers, "--"+string(mutable.Key())) } } return fmt.Errorf("no writeable configuration provider available, specify one of %s", strings.Join(writers, ", ")) @@ -52,12 +63,20 @@ func (m *Manager) Mutable() error { // Get a configuration value from the active providers. // // "value" must be a pointer to a Go type that can be unmarshalled from JSON. -func (m *Manager) Get(ctx context.Context, ref Ref, value any) error { +func (m *Manager[K]) Get(ctx context.Context, ref Ref, value any) error { key, err := m.resolver.Get(ctx, ref) - if err != nil { + // Try again at the global scope if the value is not found in module scope. + if ref.Module.Ok() && errors.Is(err, ErrNotFound) { + gref := ref + gref.Module = optional.None[string]() + key, err = m.resolver.Get(ctx, gref) + if err != nil { + return err + } + } else if err != nil { return err } - provider, ok := m.providers[key.Scheme] + provider, ok := m.providers[K(key.Scheme)] if !ok { return fmt.Errorf("no provider for scheme %q", key.Scheme) } @@ -71,7 +90,7 @@ func (m *Manager) Get(ctx context.Context, ref Ref, value any) error { // Set a configuration value in the active writing provider. // // "value" must be a Go type that can be marshalled to JSON. -func (m *Manager) Set(ctx context.Context, ref Ref, value any) error { +func (m *Manager[K]) Set(ctx context.Context, ref Ref, value any) error { if err := m.Mutable(); err != nil { return err } @@ -87,9 +106,9 @@ func (m *Manager) Set(ctx context.Context, ref Ref, value any) error { } // Unset a configuration value in all providers. -func (m *Manager) Unset(ctx context.Context, ref Ref) error { +func (m *Manager[K]) Unset(ctx context.Context, ref Ref) error { for _, provider := range m.providers { - if mutable, ok := provider.(MutableProvider); ok { + if mutable, ok := provider.(MutableProvider[K]); ok { if err := mutable.Delete(ctx, ref); err != nil && !errors.Is(err, ErrNotFound) { return err } @@ -98,21 +117,6 @@ func (m *Manager) Unset(ctx context.Context, ref Ref) error { return m.resolver.Unset(ctx, ref) } -func (m *Manager) List(ctx context.Context) ([]Entry, error) { - entries := []Entry{} - for _, provider := range m.providers { - if resolver, ok := provider.(Resolver); ok { - subentries, err := resolver.List(ctx) - if err != nil { - return nil, fmt.Errorf("%s: %w", provider.Key(), err) - } - entries = append(entries, subentries...) - } - } - subentries, err := m.resolver.List(ctx) - if err != nil { - return nil, err - } - entries = append(entries, subentries...) - return entries, nil +func (m *Manager[K]) List(ctx context.Context) ([]Entry, error) { + return m.resolver.List(ctx) } diff --git a/common/configuration/manager_test.go b/common/configuration/manager_test.go index aafb356525..0d035b62f3 100644 --- a/common/configuration/manager_test.go +++ b/common/configuration/manager_test.go @@ -23,26 +23,53 @@ func TestManager(t *testing.T) { assert.NoError(t, err) ctx := log.ContextWithNewDefaultLogger(context.Background()) - cf, err := New(ctx, - ProjectConfigResolver[FromConfig]{Config: config}, - []Provider{ - EnvarProvider[EnvarTypeConfig]{}, - InlineProvider{Inline: true}, // Writer - KeychainProvider{}, + + t.Run("Secrets", func(t *testing.T) { + kcp := KeychainProvider{Keychain: true} + _, err := kcp.Store(ctx, Ref{Name: "mutable"}, []byte("hello")) + assert.NoError(t, err) + cf, err := New(ctx, + ProjectConfigResolver[Secrets]{Config: config}, + []Provider[Secrets]{ + EnvarProvider[Secrets]{}, + InlineProvider[Secrets]{}, + kcp, + }) + assert.NoError(t, err) + testManager(t, ctx, cf, "FTL_SECRET_YmF6", []Entry{ + {Ref: Ref{Name: "baz"}, Accessor: URL("envar://baz")}, + {Ref: Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")}, + {Ref: Ref{Name: "mutable"}, Accessor: URL("keychain://mutable")}, }) - assert.NoError(t, err) + }) + t.Run("Configuration", func(t *testing.T) { + cf, err := New(ctx, + ProjectConfigResolver[Configuration]{Config: config}, + []Provider[Configuration]{ + EnvarProvider[Configuration]{}, + InlineProvider[Configuration]{Inline: true}, // Writer + }) + assert.NoError(t, err) + testManager(t, ctx, cf, "FTL_CONFIG_YmF6", []Entry{ + {Ref: Ref{Name: "baz"}, Accessor: URL("envar://baz")}, + {Ref: Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")}, + {Ref: Ref{Name: "mutable"}, Accessor: URL("inline://ImhlbGxvIg")}, + }) + }) +} - actual, err := cf.List(ctx) +// nolint +func testManager[K Kind]( + t *testing.T, + ctx context.Context, + cf *Manager[K], + envarName string, + expectedListing []Entry, +) { + actualListing, err := cf.List(ctx) assert.NoError(t, err) - expected := []Entry{ - {Ref: Ref{Name: "baz"}, Accessor: URL("envar://baz")}, - {Ref: Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")}, - {Ref: Ref{Name: "keychain"}, Accessor: URL("keychain://keychain")}, - } - - assert.Equal(t, expected, actual) - + assert.Equal(t, expectedListing, actualListing) // Try to get value from missing envar var bazValue map[string]string @@ -50,7 +77,7 @@ func TestManager(t *testing.T) { assert.IsError(t, err, ErrNotFound) // Set the envar and try again. - t.Setenv("FTL_CONFIG_YmF6", "eyJiYXoiOiJ3YXoifQ") // baz={"baz": "waz"} + t.Setenv(envarName, "eyJiYXoiOiJ3YXoifQ") // baz={"baz": "waz"} err = cf.Get(ctx, Ref{Name: "baz"}, &bazValue) assert.NoError(t, err) @@ -65,13 +92,17 @@ func TestManager(t *testing.T) { assert.IsError(t, err, ErrNotFound) // Change value. - err = cf.Set(ctx, Ref{Name: "foo"}, "hello") + err = cf.Set(ctx, Ref{Name: "mutable"}, "hello") assert.NoError(t, err) - err = cf.Get(ctx, Ref{Name: "foo"}, &fooValue) + err = cf.Get(ctx, Ref{Name: "mutable"}, &fooValue) assert.NoError(t, err) assert.Equal(t, "hello", fooValue) + actualListing, err = cf.List(ctx) + assert.NoError(t, err) + assert.Equal(t, expectedListing, actualListing) + // Delete value err = cf.Unset(ctx, Ref{Name: "foo"}) assert.NoError(t, err) diff --git a/common/configuration/projectconfig.go b/common/configuration/projectconfig.go index ec16f05d9a..476d54ba15 100644 --- a/common/configuration/projectconfig.go +++ b/common/configuration/projectconfig.go @@ -14,39 +14,22 @@ import ( pc "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal" + "github.com/TBD54566975/ftl/internal/log" ) -type FromConfigOrSecrets interface { - get(config pc.ConfigAndSecrets) map[string]*pc.URL - set(config *pc.ConfigAndSecrets, mapping map[string]*pc.URL) -} - -type FromConfig struct{} - -func (f FromConfig) get(config pc.ConfigAndSecrets) map[string]*pc.URL { return config.Config } - -func (f FromConfig) set(config *pc.ConfigAndSecrets, mapping map[string]*pc.URL) { - config.Config = mapping -} - -type FromSecrets struct{} - -func (f FromSecrets) get(config pc.ConfigAndSecrets) map[string]*pc.URL { return config.Secrets } - -func (f FromSecrets) set(config *pc.ConfigAndSecrets, mapping map[string]*pc.URL) { - config.Secrets = mapping -} - // ProjectConfigResolver is parametric Resolver that loads values from either a // project's configuration or secrets maps based on the type parameter. -type ProjectConfigResolver[From FromConfigOrSecrets] struct { - Config string `help:"Load project configuration from TOML file." placeholder:"FILE" type:"existingfile"` +// +// See the [projectconfig] package for details on the configuration file format. +type ProjectConfigResolver[K Kind] struct { + Config string `help:"Load project configuration file." placeholder:"FILE" type:"existingfile" env:"FTL_CONFIG"` } -var _ Resolver = (*ProjectConfigResolver[FromConfig])(nil) +var _ Resolver = ProjectConfigResolver[Configuration]{} +var _ Resolver = ProjectConfigResolver[Secrets]{} -func (p ProjectConfigResolver[T]) Get(ctx context.Context, ref Ref) (*url.URL, error) { - mapping, err := p.getMapping(ref.Module) +func (p ProjectConfigResolver[K]) Get(ctx context.Context, ref Ref) (*url.URL, error) { + mapping, err := p.getMapping(ctx, ref.Module) if err != nil { return nil, err } @@ -57,8 +40,8 @@ func (p ProjectConfigResolver[T]) Get(ctx context.Context, ref Ref) (*url.URL, e return (*url.URL)(key), nil } -func (p ProjectConfigResolver[T]) List(ctx context.Context) ([]Entry, error) { - config, err := p.loadConfig() +func (p ProjectConfigResolver[K]) List(ctx context.Context) ([]Entry, error) { + config, err := p.loadConfig(ctx) if err != nil { return nil, err } @@ -67,7 +50,7 @@ func (p ProjectConfigResolver[T]) List(ctx context.Context) ([]Entry, error) { moduleNames = append(moduleNames, "") for _, moduleName := range moduleNames { module := optional.Zero(moduleName) - mapping, err := p.getMapping(module) + mapping, err := p.getMapping(ctx, module) if err != nil { return nil, err } @@ -78,7 +61,7 @@ func (p ProjectConfigResolver[T]) List(ctx context.Context) ([]Entry, error) { }) } } - sort.Slice(entries, func(i, j int) bool { + sort.SliceStable(entries, func(i, j int) bool { im, _ := entries[i].Module.Get() jm, _ := entries[j].Module.Get() return im < jm || (im == jm && entries[i].Name < entries[j].Name) @@ -86,33 +69,35 @@ func (p ProjectConfigResolver[T]) List(ctx context.Context) ([]Entry, error) { return entries, nil } -func (p ProjectConfigResolver[T]) Set(ctx context.Context, ref Ref, key *url.URL) error { - mapping, err := p.getMapping(ref.Module) +func (p ProjectConfigResolver[K]) Set(ctx context.Context, ref Ref, key *url.URL) error { + mapping, err := p.getMapping(ctx, ref.Module) if err != nil { return err } mapping[ref.Name] = (*pc.URL)(key) - return p.setMapping(ref.Module, mapping) + return p.setMapping(ctx, ref.Module, mapping) } func (p ProjectConfigResolver[From]) Unset(ctx context.Context, ref Ref) error { - mapping, err := p.getMapping(ref.Module) + mapping, err := p.getMapping(ctx, ref.Module) if err != nil { return err } delete(mapping, ref.Name) - return p.setMapping(ref.Module, mapping) + return p.setMapping(ctx, ref.Module, mapping) } -func (p ProjectConfigResolver[T]) configPath() string { +func (p ProjectConfigResolver[K]) configPath() string { if p.Config != "" { return p.Config } - return filepath.Join(internal.GitRoot("."), "ftl-project.toml") + return filepath.Join(internal.GitRoot(""), "ftl-project.toml") } -func (p ProjectConfigResolver[T]) loadConfig() (pc.Config, error) { +func (p ProjectConfigResolver[K]) loadConfig(ctx context.Context) (pc.Config, error) { + logger := log.FromContext(ctx) configPath := p.configPath() + logger.Tracef("Loading config from %s", configPath) config, err := pc.Load(configPath) if errors.Is(err, os.ErrNotExist) { return pc.Config{}, nil @@ -122,40 +107,58 @@ func (p ProjectConfigResolver[T]) loadConfig() (pc.Config, error) { return config, nil } -func (p ProjectConfigResolver[T]) getMapping(module optional.Option[string]) (map[string]*pc.URL, error) { - config, err := p.loadConfig() +func (p ProjectConfigResolver[K]) getMapping(ctx context.Context, module optional.Option[string]) (map[string]*pc.URL, error) { + config, err := p.loadConfig(ctx) if err != nil { return nil, err } - var t T + + var k K + get := func(dest pc.ConfigAndSecrets) map[string]*pc.URL { + switch any(k).(type) { + case Configuration: + return dest.Config + case Secrets: + return dest.Secrets + default: + panic("unsupported kind") + } + } + if m, ok := module.Get(); ok { if config.Modules == nil { return map[string]*pc.URL{}, nil } - return t.get(config.Modules[m]), nil - } - mapping := t.get(config.Global) - if mapping == nil { - mapping = map[string]*pc.URL{} + return get(config.Modules[m]), nil } - return mapping, nil + return get(config.Global), nil } -func (p ProjectConfigResolver[T]) setMapping(module optional.Option[string], mapping map[string]*pc.URL) error { - config, err := p.loadConfig() +func (p ProjectConfigResolver[K]) setMapping(ctx context.Context, module optional.Option[string], mapping map[string]*pc.URL) error { + config, err := p.loadConfig(ctx) if err != nil { return err } - var t T + + var k K + set := func(dest *pc.ConfigAndSecrets, mapping map[string]*pc.URL) { + switch any(k).(type) { + case Configuration: + dest.Config = mapping + case Secrets: + dest.Secrets = mapping + } + } + if m, ok := module.Get(); ok { if config.Modules == nil { config.Modules = map[string]pc.ConfigAndSecrets{} } moduleConfig := config.Modules[m] - t.set(&moduleConfig, mapping) + set(&moduleConfig, mapping) config.Modules[m] = moduleConfig } else { - t.set(&config.Global, mapping) + set(&config.Global, mapping) } return pc.Save(p.configPath(), config) } diff --git a/common/configuration/testdata/ftl-project.toml b/common/configuration/testdata/ftl-project.toml index 2d7a31cc7f..94a29f2ba9 100644 --- a/common/configuration/testdata/ftl-project.toml +++ b/common/configuration/testdata/ftl-project.toml @@ -1,5 +1,10 @@ [global] +[global.secrets] +baz = "envar://baz" +foo = "inline://ImJhciI" +mutable = "keychain://mutable" + [global.configuration] baz = "envar://baz" foo = "inline://ImJhciI" -keychain = "keychain://keychain" +mutable = "inline://ImhlbGxvIg" diff --git a/examples/go/echo/echo.go b/examples/go/echo/echo.go index 9a8b47b827..a831639c34 100644 --- a/examples/go/echo/echo.go +++ b/examples/go/echo/echo.go @@ -12,6 +12,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/ftl" ) +var defaultName = ftl.Config[string]("default") + // An echo request. type EchoRequest struct { Name ftl.Option[string] `json:"name"` @@ -30,5 +32,5 @@ func Echo(ctx context.Context, req EchoRequest) (EchoResponse, error) { if err != nil { return EchoResponse{}, err } - return EchoResponse{Message: fmt.Sprintf("Hello, %s!!! It is %s!", req.Name.Default("anonymous"), tresp.Time)}, nil + return EchoResponse{Message: fmt.Sprintf("Hello, %s!!! It is %s!", req.Name.Default(defaultName.Get(ctx)), tresp.Time)}, nil } diff --git a/examples/go/echo/go.mod b/examples/go/echo/go.mod index 95a9341f43..bee24467b1 100644 --- a/examples/go/echo/go.mod +++ b/examples/go/echo/go.mod @@ -10,7 +10,10 @@ require ( connectrpc.com/connect v1.15.0 // indirect connectrpc.com/grpcreflect v1.2.0 // indirect connectrpc.com/otelconnect v0.7.0 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/TBD54566975/scaffolder v0.8.0 // indirect github.com/alecthomas/concurrency v0.0.2 // indirect + github.com/alecthomas/kong v0.8.1 // indirect github.com/alecthomas/participle/v2 v2.1.1 // indirect github.com/alecthomas/types v0.13.0 // indirect github.com/alessio/shellescape v1.4.2 // indirect diff --git a/examples/go/echo/go.sum b/examples/go/echo/go.sum index 81dd8e2438..f0a092c5ac 100644 --- a/examples/go/echo/go.sum +++ b/examples/go/echo/go.sum @@ -4,10 +4,16 @@ connectrpc.com/grpcreflect v1.2.0 h1:Q6og1S7HinmtbEuBvARLNwYmTbhEGRpHDhqrPNlmK+U connectrpc.com/grpcreflect v1.2.0/go.mod h1:nwSOKmE8nU5u/CidgHtPYk1PFI3U9ignz7iDMxOYkSY= connectrpc.com/otelconnect v0.7.0 h1:ZH55ZZtcJOTKWWLy3qmL4Pam4RzRWBJFOqTPyAqCXkY= connectrpc.com/otelconnect v0.7.0/go.mod h1:Bt2ivBymHZHqxvo4HkJ0EwHuUzQN6k2l0oH+mp/8nwc= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/TBD54566975/scaffolder v0.8.0 h1:DWl1K3dWcLsOPAYGQGPQXtffrml6XCB0tF05JdpMqZU= +github.com/TBD54566975/scaffolder v0.8.0/go.mod h1:Ab/jbQ4q8EloYL0nbkdh2DVvkGc4nxr1OcIbdMpTxxg= github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/concurrency v0.0.2 h1:Q3kGPtLbleMbH9lHX5OBFvJygfyFw29bXZKBg+IEVuo= github.com/alecthomas/concurrency v0.0.2/go.mod h1:GmuQb/iHX7mbNtPlC/WDzEFxDMB0HYFer2Qda9QTs7w= +github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= +github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8= github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= diff --git a/ftl-project.toml b/ftl-project.toml new file mode 100644 index 0000000000..eafec15db4 --- /dev/null +++ b/ftl-project.toml @@ -0,0 +1,6 @@ +[global] + +[modules] + [modules.echo] + [modules.echo.configuration] + default = "inline://ImFub255bW91cyI" diff --git a/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl b/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl index d9100a0f09..56b07af1f7 100644 --- a/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl +++ b/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl @@ -5,3 +5,7 @@ go {{ .GoVersion }} {{ if ne .FTLVersion "" }} require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{ end }} + +{{- range .Replacements }} +replace {{ .Old }} => {{ .New }} +{{- end }} \ No newline at end of file diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index ef24bf1013..21700a1879 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -4,13 +4,15 @@ import ( "context" "fmt" "maps" + "net" "os" "path" "path/filepath" - "reflect" + stdreflect "reflect" "strings" "github.com/TBD54566975/scaffolder" + "golang.design/x/reflect" "golang.org/x/mod/modfile" "google.golang.org/protobuf/proto" @@ -25,9 +27,10 @@ import ( type externalModuleContext struct { ModuleDir string *schema.Schema - GoVersion string - FTLVersion string - Main string + GoVersion string + FTLVersion string + Main string + Replacements []*modfile.Replace } type goVerb struct { @@ -35,10 +38,11 @@ type goVerb struct { } type mainModuleContext struct { - GoVersion string - FTLVersion string - Name string - Verbs []goVerb + GoVersion string + FTLVersion string + Name string + Verbs []goVerb + Replacements []*modfile.Replace } func (b externalModuleContext) NonMainModules() []*schema.Module { @@ -56,7 +60,7 @@ const buildDirName = "_ftl" // Build the given module. func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { - goModVersion, err := updateGoModule(filepath.Join(moduleDir, "go.mod")) + replacements, goModVersion, err := updateGoModule(filepath.Join(moduleDir, "go.mod")) if err != nil { return err } @@ -81,11 +85,12 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { logger.Debugf("Generating external modules") if err := internal.ScaffoldZip(externalModuleTemplateFiles(), moduleDir, externalModuleContext{ - ModuleDir: moduleDir, - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Schema: sch, - Main: config.Module, + ModuleDir: moduleDir, + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Schema: sch, + Main: config.Module, + Replacements: replacements, }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -121,10 +126,11 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { } } if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{ - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Name: main.Name, - Verbs: goVerbs, + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Name: main.Name, + Verbs: goVerbs, + Replacements: replacements, }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -138,6 +144,11 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } +func online() bool { + _, err := net.LookupHost("proxy.golang.org") + return err == nil +} + var scaffoldFuncs = scaffolder.FuncMap{ "comment": func(s []string) string { if len(s) == 0 { @@ -147,7 +158,7 @@ var scaffoldFuncs = scaffolder.FuncMap{ }, "type": genType, "is": func(kind string, t schema.Node) bool { - return reflect.Indirect(reflect.ValueOf(t)).Type().Name() == kind + return stdreflect.Indirect(stdreflect.ValueOf(t)).Type().Name() == kind }, "imports": func(m *schema.Module) map[string]string { imports := map[string]string{} @@ -242,38 +253,53 @@ func genType(module *schema.Module, t schema.Type) string { panic(fmt.Sprintf("unsupported type %T", t)) } -// Update go.mod file to include the FTL version. -func updateGoModule(goModPath string) (string, error) { +// Update go.mod file to include the FTL version and return the Go version and any replace directives. +func updateGoModule(goModPath string) (replacements []*modfile.Replace, goVersion string, err error) { goModBytes, err := os.ReadFile(goModPath) if err != nil { - return "", fmt.Errorf("failed to read %s: %w", goModPath, err) + return nil, "", fmt.Errorf("failed to read %s: %w", goModPath, err) } goModFile, err := modfile.Parse(goModPath, goModBytes, nil) if err != nil { - return "", fmt.Errorf("failed to parse %s: %w", goModPath, err) + return nil, "", fmt.Errorf("failed to parse %s: %w", goModPath, err) + } + + // Propagate any replace directives. + replacements = reflect.DeepCopy(goModFile.Replace) + for i, r := range replacements { + if strings.HasPrefix(r.New.Path, ".") { + abs, err := filepath.Abs(filepath.Join(filepath.Dir(goModPath), r.New.Path)) + if err != nil { + return nil, "", err + } + replacements[i].New.Path = abs + } } + + // Early return if we're not updating anything. if !ftl.IsRelease(ftl.Version) || !shouldUpdateVersion(goModFile) { - return goModFile.Go.Version, nil + return replacements, goModFile.Go.Version, nil } + if err := goModFile.AddRequire("github.com/TBD54566975/ftl", "v"+ftl.Version); err != nil { - return "", fmt.Errorf("failed to add github.com/TBD54566975/ftl to %s: %w", goModPath, err) + return nil, "", fmt.Errorf("failed to add github.com/TBD54566975/ftl to %s: %w", goModPath, err) } // Atomically write the updated go.mod file. tmpFile, err := os.CreateTemp(filepath.Dir(goModPath), ".go.mod-") if err != nil { - return "", fmt.Errorf("update %s: %w", goModPath, err) + return nil, "", fmt.Errorf("update %s: %w", goModPath, err) } defer os.Remove(tmpFile.Name()) // Delete the temp file if we error. defer tmpFile.Close() goModBytes = modfile.Format(goModFile.Syntax) if _, err := tmpFile.Write(goModBytes); err != nil { - return "", fmt.Errorf("update %s: %w", goModPath, err) + return nil, "", fmt.Errorf("update %s: %w", goModPath, err) } if err := os.Rename(tmpFile.Name(), goModPath); err != nil { - return "", fmt.Errorf("update %s: %w", goModPath, err) + return nil, "", fmt.Errorf("update %s: %w", goModPath, err) } - return goModFile.Go.Version, nil + return replacements, goModFile.Go.Version, nil } func shouldUpdateVersion(goModfile *modfile.File) bool { diff --git a/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl b/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl index ce891c2615..2f1d05e47d 100644 --- a/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl +++ b/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl @@ -1,3 +1,11 @@ module ftl go {{ .GoVersion }} + +{{ if ne .FTLVersion "" }} +require github.com/TBD54566975/ftl v{{ .FTLVersion }} +{{ end }} + +{{- range .Replacements }} +replace {{ .Old }} => {{ .New }} +{{- end }} \ No newline at end of file diff --git a/go-runtime/ftl/config.go b/go-runtime/ftl/config.go index debd42794d..ec2d187668 100644 --- a/go-runtime/ftl/config.go +++ b/go-runtime/ftl/config.go @@ -1,11 +1,12 @@ package ftl import ( - "encoding/json" + "context" "fmt" - "os" "runtime" "strings" + + "github.com/TBD54566975/ftl/common/configuration" ) // ConfigType is a type that can be used as a configuration value. @@ -28,16 +29,11 @@ func (c *ConfigValue[T]) String() string { } // Get returns the value of the configuration key from FTL. -func (c *ConfigValue[T]) Get() (out T) { - value, ok := os.LookupEnv(fmt.Sprintf("FTL_CONFIG_%s_%s", strings.ToUpper(c.module), strings.ToUpper(c.name))) - if !ok { - value, ok = os.LookupEnv(fmt.Sprintf("FTL_CONFIG_%s", strings.ToUpper(c.name))) - if !ok { - return out - } - } - if err := json.Unmarshal([]byte(value), &out); err != nil { - panic(fmt.Errorf("failed to parse %s value %q: %w", c, value, err)) +func (c *ConfigValue[T]) Get(ctx context.Context) (out T) { + cm := configuration.ConfigFromContext(ctx) + err := cm.Get(ctx, configuration.NewRef(c.module, c.name), &out) + if err != nil { + panic(fmt.Errorf("failed to get %s: %w", c, err)) } return } diff --git a/go-runtime/ftl/config_test.go b/go-runtime/ftl/config_test.go index 75b269da88..bffbf56303 100644 --- a/go-runtime/ftl/config_test.go +++ b/go-runtime/ftl/config_test.go @@ -1,17 +1,24 @@ package ftl import ( + "context" "testing" "github.com/alecthomas/assert/v2" + + "github.com/TBD54566975/ftl/common/configuration" + "github.com/TBD54566975/ftl/internal/log" ) func TestConfig(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + cm, err := configuration.NewConfigurationManager(ctx, "testdata/ftl-project.toml") + assert.NoError(t, err) + ctx = configuration.ContextWithConfig(ctx, cm) type C struct { One string Two string } - t.Setenv("FTL_CONFIG_TESTING_TEST", `{"one": "one", "two": "two"}`) config := Config[C]("test") - assert.Equal(t, C{"one", "two"}, config.Get()) + assert.Equal(t, C{"one", "two"}, config.Get(ctx)) } diff --git a/go-runtime/ftl/secrets.go b/go-runtime/ftl/secrets.go index 79d628ea76..109f1a575a 100644 --- a/go-runtime/ftl/secrets.go +++ b/go-runtime/ftl/secrets.go @@ -1,10 +1,10 @@ package ftl import ( - "encoding/json" + "context" "fmt" - "os" - "strings" + + "github.com/TBD54566975/ftl/common/configuration" ) // SecretType is a type that can be used as a secret value. @@ -27,16 +27,10 @@ func (s *SecretValue[Type]) String() string { } // Get returns the value of the secret from FTL. -func (s *SecretValue[Type]) Get() (out Type) { - value, ok := os.LookupEnv(fmt.Sprintf("FTL_SECRET_%s_%s", strings.ToUpper(s.module), strings.ToUpper(s.name))) - if !ok { - value, ok = os.LookupEnv(fmt.Sprintf("FTL_SECRET_%s", strings.ToUpper(s.name))) - if !ok { - return out - } - } - if err := json.Unmarshal([]byte(value), &out); err != nil { - panic(fmt.Errorf("failed to parse %s: %w", s, err)) +func (s *SecretValue[Type]) Get(ctx context.Context) (out Type) { + sm := configuration.SecretsFromContext(ctx) + if err := sm.Get(ctx, configuration.NewRef(s.module, s.name), &out); err != nil { + panic(fmt.Errorf("failed to get %s: %w", s, err)) } return } diff --git a/go-runtime/ftl/secrets_test.go b/go-runtime/ftl/secrets_test.go index b012295693..4874f48657 100644 --- a/go-runtime/ftl/secrets_test.go +++ b/go-runtime/ftl/secrets_test.go @@ -1,17 +1,24 @@ package ftl import ( + "context" "testing" "github.com/alecthomas/assert/v2" + + "github.com/TBD54566975/ftl/common/configuration" + "github.com/TBD54566975/ftl/internal/log" ) func TestSecret(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + sm, err := configuration.NewSecretsManager(ctx, "testdata/ftl-project.toml") + assert.NoError(t, err) + ctx = configuration.ContextWithSecrets(ctx, sm) type C struct { One string Two string } - t.Setenv("FTL_SECRET_TESTING_TEST", `{"one": "one", "two": "two"}`) - config := Secret[C]("test") - assert.Equal(t, C{"one", "two"}, config.Get()) + config := Secret[C]("secret") + assert.Equal(t, C{"one", "two"}, config.Get(ctx)) } diff --git a/go-runtime/ftl/testdata/ftl-project.toml b/go-runtime/ftl/testdata/ftl-project.toml new file mode 100644 index 0000000000..71ef4bf259 --- /dev/null +++ b/go-runtime/ftl/testdata/ftl-project.toml @@ -0,0 +1,8 @@ +[global] + +[modules] + [modules.testing] + [modules.testing.configuration] + test = "inline://eyJvbmUiOiJvbmUiLCJ0d28iOiJ0d28ifQ" + [modules.testing.secrets] + secret = "inline://eyJvbmUiOiJvbmUiLCJ0d28iOiJ0d28ifQ" diff --git a/go-runtime/server/server.go b/go-runtime/server/server.go index a2117f19dc..868e34f6a4 100644 --- a/go-runtime/server/server.go +++ b/go-runtime/server/server.go @@ -10,9 +10,10 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" + cf "github.com/TBD54566975/ftl/common/configuration" "github.com/TBD54566975/ftl/common/plugin" "github.com/TBD54566975/ftl/go-runtime/encoding" - sdkgo "github.com/TBD54566975/ftl/go-runtime/ftl" + "github.com/TBD54566975/ftl/go-runtime/ftl" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/maps" "github.com/TBD54566975/ftl/internal/observability" @@ -22,6 +23,8 @@ import ( type UserVerbConfig struct { FTLEndpoint *url.URL `help:"FTL endpoint." env:"FTL_ENDPOINT" required:""` ObservabilityConfig observability.Config `embed:"" prefix:"o11y-"` + + Config string `help:"Load project configuration file." placeholder:"FILE" type:"existingfile" env:"FTL_CONFIG"` } // NewUserVerbServer starts a new code-generated drive for user Verbs. @@ -30,26 +33,39 @@ type UserVerbConfig struct { func NewUserVerbServer(moduleName string, handlers ...Handler) plugin.Constructor[ftlv1connect.VerbServiceHandler, UserVerbConfig] { return func(ctx context.Context, uc UserVerbConfig) (context.Context, ftlv1connect.VerbServiceHandler, error) { verbServiceClient := rpc.Dial(ftlv1connect.NewVerbServiceClient, uc.FTLEndpoint.String(), log.Error) + // Add config manager to context. ctx = rpc.ContextWithClient(ctx, verbServiceClient) + cm, err := cf.NewConfigurationManager(ctx, uc.Config) + if err != nil { + return nil, nil, err + } + ctx = cf.ContextWithConfig(ctx, cm) + + // Add secrets manager to context. + sm, err := cf.NewSecretsManager(ctx, uc.Config) + if err != nil { + return nil, nil, err + } + ctx = cf.ContextWithSecrets(ctx, sm) - err := observability.Init(ctx, moduleName, "HEAD", uc.ObservabilityConfig) + err = observability.Init(ctx, moduleName, "HEAD", uc.ObservabilityConfig) if err != nil { return nil, nil, err } - hmap := maps.FromSlice(handlers, func(h Handler) (sdkgo.VerbRef, Handler) { return h.ref, h }) + hmap := maps.FromSlice(handlers, func(h Handler) (ftl.VerbRef, Handler) { return h.ref, h }) return ctx, &moduleServer{handlers: hmap}, nil } } // Handler for a Verb. type Handler struct { - ref sdkgo.VerbRef + ref ftl.VerbRef fn func(ctx context.Context, req []byte) ([]byte, error) } // Handle creates a Handler from a Verb. func Handle[Req, Resp any](verb func(ctx context.Context, req Req) (Resp, error)) Handler { - ref := sdkgo.VerbToRef(verb) + ref := ftl.VerbToRef(verb) return Handler{ ref: ref, fn: func(ctx context.Context, reqdata []byte) ([]byte, error) { @@ -80,7 +96,7 @@ var _ ftlv1connect.VerbServiceHandler = (*moduleServer)(nil) // This is the server that is compiled into the same binary as user-defined Verbs. type moduleServer struct { - handlers map[sdkgo.VerbRef]Handler + handlers map[ftl.VerbRef]Handler } func (m *moduleServer) Call(ctx context.Context, req *connect.Request[ftlv1.CallRequest]) (response *connect.Response[ftlv1.CallResponse], err error) { @@ -102,7 +118,7 @@ func (m *moduleServer) Call(ctx context.Context, req *connect.Request[ftlv1.Call }}}) } }() - handler, ok := m.handlers[sdkgo.VerbRefFromProto(req.Msg.Verb)] + handler, ok := m.handlers[ftl.VerbRefFromProto(req.Msg.Verb)] if !ok { return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("verb %q not found", req.Msg.Verb)) } diff --git a/internal/observability/client.go b/internal/observability/client.go index 07c78faaa5..bb6174c5ba 100644 --- a/internal/observability/client.go +++ b/internal/observability/client.go @@ -19,18 +19,18 @@ import ( const schemaURL = semconv.SchemaURL -type exportOTELFlag bool +type ExportOTELFlag bool -// Default behaviour of Kong is to use strconv.ParseBool, but we want to be less strict. -func (e *exportOTELFlag) UnmarshalText(text []byte) error { +func (e *ExportOTELFlag) UnmarshalText(text []byte) error { + // Default behaviour of Kong is to use strconv.ParseBool, but we want to be less strict. v := strings.ToLower(string(text)) - *e = exportOTELFlag(!(v == "false" || v == "0" || v == "no" || v == "")) + *e = ExportOTELFlag(!(v == "false" || v == "0" || v == "no" || v == "")) return nil } type Config struct { LogLevel log.Level `default:"error" help:"OTEL log level." env:"FTL_O11Y_LOG_LEVEL"` - ExportOTEL exportOTELFlag `help:"Export observability data to OTEL." env:"OTEL_EXPORTER_OTLP_ENDPOINT"` + ExportOTEL ExportOTELFlag `help:"Export observability data to OTEL." env:"OTEL_EXPORTER_OTLP_ENDPOINT"` } func Init(ctx context.Context, serviceName, serviceVersion string, config Config) error { diff --git a/scripts/autofmt b/scripts/autofmt index 5fd4cd1204..da5095cf26 100755 --- a/scripts/autofmt +++ b/scripts/autofmt @@ -2,7 +2,7 @@ set -euo pipefail # Run go mod tidy on all modules -find . -name 'go.mod' ! -path '*/.ftl/*' ! -path '*/scaffolding/*' -print0 | xargs -0 dirname | while read -r dir; do (echo "Tidying Go module in '$dir'..."; cd "$dir" && go mod tidy); done +find . -name 'go.mod' ! -path '*/_*/*' ! -path '*/.ftl/*' ! -path '*/scaffolding/*' -print0 | xargs -0 dirname | while read -r dir; do (echo "Tidying Go module in '$dir'..."; cd "$dir" && go mod tidy); done echo "Formatting Go..." # shellcheck disable=SC2207