From eb2722bda8a13bb2aac1bdd49245ada5744c8f48 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Tue, 10 Sep 2024 15:35:08 +1000 Subject: [PATCH] feat: start of profile support (#2639) This adds the beginnings of a high-level package for initialising a project, saving and loading profiles. There's also a lower-level API over the storage mechanism. Design: https://hackmd.io/@ftl/Sy2GtZKnR --- backend/controller/admin/admin.go | 21 +- internal/configuration/api.go | 4 +- internal/configuration/manager/cache.go | 10 +- internal/configuration/manager/manager.go | 16 +- .../configuration/manager/manager_test.go | 6 +- .../providers/1password_provider.go | 22 +- internal/configuration/providers/asm.go | 16 +- .../providers/db_config_provider.go | 12 +- .../configuration/providers/envar_provider.go | 16 +- .../providers/inline_provider.go | 18 +- .../providers/keychain_provider.go | 18 +- .../providerstest/manual_sync_utils.go | 2 +- internal/configuration/providers/registry.go | 34 +++ internal/configuration/routers/file_router.go | 127 +++++++++++ .../configuration/routers/file_router_test.go | 52 +++++ internal/profiles/internal/persistence.go | 199 ++++++++++++++++++ internal/profiles/profiles.go | 107 ++++++++++ internal/profiles/profiles_test.go | 63 ++++++ internal/projectconfig/projectconfig.go | 16 +- 19 files changed, 705 insertions(+), 54 deletions(-) create mode 100644 internal/configuration/providers/registry.go create mode 100644 internal/configuration/routers/file_router.go create mode 100644 internal/configuration/routers/file_router_test.go create mode 100644 internal/profiles/internal/persistence.go create mode 100644 internal/profiles/profiles.go create mode 100644 internal/profiles/profiles_test.go diff --git a/backend/controller/admin/admin.go b/backend/controller/admin/admin.go index 16da685b25..021e865f9d 100644 --- a/backend/controller/admin/admin.go +++ b/backend/controller/admin/admin.go @@ -13,6 +13,7 @@ import ( "github.com/TBD54566975/ftl/go-runtime/encoding" "github.com/TBD54566975/ftl/internal/configuration" "github.com/TBD54566975/ftl/internal/configuration/manager" + "github.com/TBD54566975/ftl/internal/configuration/providers" "github.com/TBD54566975/ftl/internal/log" ) @@ -94,17 +95,17 @@ func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1 return connect.NewResponse(&ftlv1.GetConfigResponse{Value: vb}), nil } -func configProviderKey(p *ftlv1.ConfigProvider) string { +func configProviderKey(p *ftlv1.ConfigProvider) configuration.ProviderKey { if p == nil { return "" } switch *p { case ftlv1.ConfigProvider_CONFIG_INLINE: - return "inline" + return providers.InlineProviderKey case ftlv1.ConfigProvider_CONFIG_ENVAR: - return "envar" + return providers.EnvarProviderKey case ftlv1.ConfigProvider_CONFIG_DB: - return "db" + return providers.DatabaseConfigProviderKey } return "" } @@ -188,21 +189,21 @@ func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1 return connect.NewResponse(&ftlv1.GetSecretResponse{Value: vb}), nil } -func secretProviderKey(p *ftlv1.SecretProvider) string { +func secretProviderKey(p *ftlv1.SecretProvider) configuration.ProviderKey { if p == nil { return "" } switch *p { case ftlv1.SecretProvider_SECRET_INLINE: - return "inline" + return providers.InlineProviderKey case ftlv1.SecretProvider_SECRET_ENVAR: - return "envar" + return providers.EnvarProviderKey case ftlv1.SecretProvider_SECRET_KEYCHAIN: - return "keychain" + return providers.KeychainProviderKey case ftlv1.SecretProvider_SECRET_OP: - return "op" + return providers.OnePasswordProviderKey case ftlv1.SecretProvider_SECRET_ASM: - return "asm" + return providers.ASMProviderKey } return "" } diff --git a/internal/configuration/api.go b/internal/configuration/api.go index 570729ab6c..f52e78ca94 100644 --- a/internal/configuration/api.go +++ b/internal/configuration/api.go @@ -103,10 +103,12 @@ type Router[R Role] interface { List(ctx context.Context) ([]Entry, error) } +type ProviderKey string + // Provider is a generic interface for storing and retrieving configuration and secrets. type Provider[R Role] interface { Role() R - Key() string + Key() ProviderKey // Store a configuration value and return its key. Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) diff --git a/internal/configuration/manager/cache.go b/internal/configuration/manager/cache.go index 74249ed766..d4fd07a3f4 100644 --- a/internal/configuration/manager/cache.go +++ b/internal/configuration/manager/cache.go @@ -27,7 +27,7 @@ type listProvider interface { } type updateCacheEvent struct { - key string + key configuration.ProviderKey ref configuration.Ref // value is nil when the value was deleted value optional.Option[[]byte] @@ -39,7 +39,7 @@ type updateCacheEvent struct { // Sync happens periodically. // Updates do not go through the cache, but the cache is notified after the update occurs. type cache[R configuration.Role] struct { - providers map[string]*cacheProvider[R] + providers map[configuration.ProviderKey]*cacheProvider[R] // list provider is used to determine which providers are expected to have values, and therefore need to be synced listProvider listProvider @@ -50,7 +50,7 @@ type cache[R configuration.Role] struct { } func newCache[R configuration.Role](ctx context.Context, providers []configuration.AsynchronousProvider[R], listProvider listProvider) *cache[R] { - cacheProviders := make(map[string]*cacheProvider[R], len(providers)) + cacheProviders := make(map[configuration.ProviderKey]*cacheProvider[R], len(providers)) for _, provider := range providers { cacheProviders[provider.Key()] = &cacheProvider[R]{ provider: provider, @@ -73,7 +73,7 @@ func newCache[R configuration.Role](ctx context.Context, providers []configurati // load is called by the manager to get a value from the cache func (c *cache[R]) load(ref configuration.Ref, key *url.URL) ([]byte, error) { providerKey := ProviderKeyForAccessor(key) - provider, ok := c.providers[key.Scheme] + provider, ok := c.providers[configuration.ProviderKey(key.Scheme)] if !ok { return nil, fmt.Errorf("no cache provider for key %q", providerKey) } @@ -103,7 +103,7 @@ func (c *cache[R]) updatedValue(ref configuration.Ref, value []byte, accessor *u } // deletedValue should be called when a value is deleted in the provider -func (c *cache[R]) deletedValue(ref configuration.Ref, pkey string) { +func (c *cache[R]) deletedValue(ref configuration.Ref, pkey configuration.ProviderKey) { if _, ok := c.providers[pkey]; !ok { // not syncing this provider return diff --git a/internal/configuration/manager/manager.go b/internal/configuration/manager/manager.go index e9b3cd6818..62c2974bf1 100644 --- a/internal/configuration/manager/manager.go +++ b/internal/configuration/manager/manager.go @@ -19,7 +19,7 @@ import ( // Manager is a high-level configuration manager that abstracts the details of // the Router and Provider interfaces. type Manager[R configuration.Role] struct { - providers map[string]configuration.Provider[R] + providers map[configuration.ProviderKey]configuration.Provider[R] router configuration.Router[R] obfuscator optional.Option[configuration.Obfuscator] cache *cache[R] @@ -49,7 +49,7 @@ func NewDefaultConfigurationManagerFromConfig(ctx context.Context, config string // New configuration manager. func New[R configuration.Role](ctx context.Context, router configuration.Router[R], providers []configuration.Provider[R]) (*Manager[R], error) { m := &Manager[R]{ - providers: map[string]configuration.Provider[R]{}, + providers: map[configuration.ProviderKey]configuration.Provider[R]{}, } for _, p := range providers { m.providers[p.Key()] = p @@ -70,8 +70,8 @@ func New[R configuration.Role](ctx context.Context, router configuration.Router[ return m, nil } -func ProviderKeyForAccessor(accessor *url.URL) string { - return accessor.Scheme +func ProviderKeyForAccessor(accessor *url.URL) configuration.ProviderKey { + return configuration.ProviderKey(accessor.Scheme) } // getData returns a data value for a configuration from the active providers. @@ -136,13 +136,13 @@ func (m *Manager[R]) Get(ctx context.Context, ref configuration.Ref, value any) func (m *Manager[R]) availableProviderKeys() []string { keys := make([]string, 0, len(m.providers)) for k := range m.providers { - keys = append(keys, "--"+k) + keys = append(keys, "--"+string(k)) } return keys } // Set a configuration value, encoding "value" as JSON before storing it. -func (m *Manager[R]) Set(ctx context.Context, pkey string, ref configuration.Ref, value any) error { +func (m *Manager[R]) Set(ctx context.Context, pkey configuration.ProviderKey, ref configuration.Ref, value any) error { data, err := json.Marshal(value) if err != nil { return err @@ -151,7 +151,7 @@ func (m *Manager[R]) Set(ctx context.Context, pkey string, ref configuration.Ref } // SetJSON sets a configuration value using raw JSON data. -func (m *Manager[R]) SetJSON(ctx context.Context, pkey string, ref configuration.Ref, value json.RawMessage) error { +func (m *Manager[R]) SetJSON(ctx context.Context, pkey configuration.ProviderKey, ref configuration.Ref, value json.RawMessage) error { if err := checkJSON(value); err != nil { return fmt.Errorf("invalid value for %s, must be JSON: %w", m.router.Role(), err) } @@ -211,7 +211,7 @@ func (m *Manager[R]) MapForModule(ctx context.Context, module string) (map[strin } // Unset a configuration value in all providers. -func (m *Manager[R]) Unset(ctx context.Context, pkey string, ref configuration.Ref) error { +func (m *Manager[R]) Unset(ctx context.Context, pkey configuration.ProviderKey, ref configuration.Ref) error { provider, ok := m.providers[pkey] if !ok { pkeys := strings.Join(m.availableProviderKeys(), ", ") diff --git a/internal/configuration/manager/manager_test.go b/internal/configuration/manager/manager_test.go index 91f1cbd8f5..92dbaa3710 100644 --- a/internal/configuration/manager/manager_test.go +++ b/internal/configuration/manager/manager_test.go @@ -35,7 +35,7 @@ func TestManager(t *testing.T) { kcp, }) assert.NoError(t, err) - testManager(t, ctx, cf, "keychain", "FTL_SECRET_YmF6", []configuration.Entry{ + testManager(t, ctx, cf, providers.KeychainProviderKey, "FTL_SECRET_YmF6", []configuration.Entry{ {Ref: configuration.Ref{Name: "baz"}, Accessor: URL("envar://baz")}, {Ref: configuration.Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")}, {Ref: configuration.Ref{Name: "mutable"}, Accessor: URL("keychain://mutable")}, @@ -49,7 +49,7 @@ func TestManager(t *testing.T) { providers.Inline[configuration.Configuration]{}, }) assert.NoError(t, err) - testManager(t, ctx, cf, "inline", "FTL_CONFIG_YmF6", []configuration.Entry{ + testManager(t, ctx, cf, providers.InlineProviderKey, "FTL_CONFIG_YmF6", []configuration.Entry{ {Ref: configuration.Ref{Name: "baz"}, Accessor: URL("envar://baz")}, {Ref: configuration.Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")}, {Ref: configuration.Ref{Name: "mutable"}, Accessor: URL("inline://ImhlbGxvIg")}, @@ -119,7 +119,7 @@ func testManager[R configuration.Role]( t *testing.T, ctx context.Context, cf *Manager[R], - providerKey string, + providerKey configuration.ProviderKey, envarName string, expectedListing []configuration.Entry, ) { diff --git a/internal/configuration/providers/1password_provider.go b/internal/configuration/providers/1password_provider.go index b489bd1a5a..80e267d44c 100644 --- a/internal/configuration/providers/1password_provider.go +++ b/internal/configuration/providers/1password_provider.go @@ -18,6 +18,8 @@ import ( "github.com/TBD54566975/ftl/internal/log" ) +const OnePasswordProviderKey configuration.ProviderKey = "op" + // OnePassword is a configuration provider that reads passwords from // 1Password vaults via the "op" command line tool. type OnePassword struct { @@ -25,10 +27,24 @@ type OnePassword struct { ProjectName string } +func NewOnePassword(vault string, projectName string) OnePassword { + return OnePassword{ + Vault: vault, + ProjectName: projectName, + } +} + +func NewOnePasswordFactory(vault string, projectName string) (configuration.ProviderKey, Factory[configuration.Secrets]) { + return OnePasswordProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Secrets], error) { + return NewOnePassword(vault, projectName), nil + } +} + +var _ configuration.Provider[configuration.Secrets] = OnePassword{} var _ configuration.AsynchronousProvider[configuration.Secrets] = OnePassword{} -func (OnePassword) Role() configuration.Secrets { return configuration.Secrets{} } -func (o OnePassword) Key() string { return "op" } +func (OnePassword) Role() configuration.Secrets { return configuration.Secrets{} } +func (o OnePassword) Key() configuration.ProviderKey { return OnePasswordProviderKey } func (o OnePassword) Delete(ctx context.Context, ref configuration.Ref) error { return nil } @@ -110,7 +126,7 @@ func (o OnePassword) Store(ctx context.Context, ref configuration.Ref, value []b return nil, fmt.Errorf("vault name %q contains invalid characters. a-z A-Z 0-9 _ . - are valid", o.Vault) } - url := &url.URL{Scheme: "op", Host: o.Vault} + url := &url.URL{Scheme: string(OnePasswordProviderKey), Host: o.Vault} // make sure item exists _, err := o.getItem(ctx, o.Vault) diff --git a/internal/configuration/providers/asm.go b/internal/configuration/providers/asm.go index 42a4b80d1b..96b4de33f7 100644 --- a/internal/configuration/providers/asm.go +++ b/internal/configuration/providers/asm.go @@ -18,6 +18,8 @@ import ( "github.com/TBD54566975/ftl/internal/rpc" ) +const ASMProviderKey configuration.ProviderKey = "asm" + type asmClient interface { name() string syncInterval() time.Duration @@ -37,6 +39,12 @@ type ASM struct { var _ configuration.AsynchronousProvider[configuration.Secrets] = &ASM{} +func NewASMFactory(secretsClient *secretsmanager.Client, advertise *url.URL, leaser leases.Leaser) (configuration.ProviderKey, Factory[configuration.Secrets]) { + return ASMProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Secrets], error) { + return NewASM(ctx, secretsClient, advertise, leaser), nil + } +} + func NewASM(ctx context.Context, secretsClient *secretsmanager.Client, advertise *url.URL, leaser leases.Leaser) *ASM { return newASMForTesting(ctx, secretsClient, advertise, leaser, optional.None[asmClient]()) } @@ -58,7 +66,7 @@ func newASMForTesting(ctx context.Context, secretsClient *secretsmanager.Client, coordinator := leader.NewCoordinator[asmClient]( ctx, advertise, - leases.SystemKey("asm"), + leases.SystemKey(string(ASMProviderKey)), leaser, time.Second*10, leaderFactory, @@ -71,7 +79,7 @@ func newASMForTesting(ctx context.Context, secretsClient *secretsmanager.Client, func asmURLForRef(ref configuration.Ref) *url.URL { return &url.URL{ - Scheme: "asm", + Scheme: string(ASMProviderKey), Host: ref.String(), } } @@ -80,8 +88,8 @@ func (ASM) Role() configuration.Secrets { return configuration.Secrets{} } -func (ASM) Key() string { - return "asm" +func (*ASM) Key() configuration.ProviderKey { + return ASMProviderKey } func (a *ASM) SyncInterval() time.Duration { diff --git a/internal/configuration/providers/db_config_provider.go b/internal/configuration/providers/db_config_provider.go index c0f5acccf1..0ec1eb294c 100644 --- a/internal/configuration/providers/db_config_provider.go +++ b/internal/configuration/providers/db_config_provider.go @@ -11,6 +11,8 @@ import ( "github.com/TBD54566975/ftl/internal/configuration" ) +const DatabaseConfigProviderKey configuration.ProviderKey = "db" + // DatabaseConfig is a configuration provider that stores configuration in its key. type DatabaseConfig struct { dal DatabaseConfigDAL @@ -24,6 +26,12 @@ type DatabaseConfigDAL interface { UnsetModuleConfiguration(ctx context.Context, module optional.Option[string], name string) error } +func NewDatabaseConfigFactory(dal DatabaseConfigDAL) (configuration.ProviderKey, Factory[configuration.Configuration]) { + return DatabaseConfigProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Configuration], error) { + return NewDatabaseConfig(dal), nil + } +} + func NewDatabaseConfig(dal DatabaseConfigDAL) DatabaseConfig { return DatabaseConfig{ dal: dal, @@ -31,7 +39,7 @@ func NewDatabaseConfig(dal DatabaseConfigDAL) DatabaseConfig { } func (DatabaseConfig) Role() configuration.Configuration { return configuration.Configuration{} } -func (DatabaseConfig) Key() string { return "db" } +func (DatabaseConfig) Key() configuration.ProviderKey { return DatabaseConfigProviderKey } func (d DatabaseConfig) Load(ctx context.Context, ref configuration.Ref, key *url.URL) ([]byte, error) { value, err := d.dal.GetModuleConfiguration(ctx, ref.Module, ref.Name) @@ -46,7 +54,7 @@ func (d DatabaseConfig) Store(ctx context.Context, ref configuration.Ref, value if err != nil { return nil, fmt.Errorf("failed to set configuration: %w", err) } - return &url.URL{Scheme: "db"}, nil + return &url.URL{Scheme: string(DatabaseConfigProviderKey)}, nil } func (d DatabaseConfig) Delete(ctx context.Context, ref configuration.Ref) error { diff --git a/internal/configuration/providers/envar_provider.go b/internal/configuration/providers/envar_provider.go index f2fed46a6f..0de5c50263 100644 --- a/internal/configuration/providers/envar_provider.go +++ b/internal/configuration/providers/envar_provider.go @@ -10,14 +10,24 @@ import ( "github.com/TBD54566975/ftl/internal/configuration" ) +const EnvarProviderKey configuration.ProviderKey = "envar" + // Envar is a configuration provider that reads secrets or configuration // from environment variables. type Envar[R configuration.Role] struct{} var _ configuration.SynchronousProvider[configuration.Configuration] = Envar[configuration.Configuration]{} -func (Envar[R]) Role() R { var r R; return r } -func (Envar[R]) Key() string { return "envar" } +func NewEnvarFactory[R configuration.Role]() (configuration.ProviderKey, Factory[R]) { + return EnvarProviderKey, func(ctx context.Context) (configuration.Provider[R], error) { + return NewEnvar[R](), nil + } +} + +func NewEnvar[R configuration.Role]() Envar[R] { return Envar[R]{} } + +func (Envar[R]) Role() R { var r R; return r } +func (Envar[R]) Key() configuration.ProviderKey { return EnvarProviderKey } func (e Envar[R]) Load(ctx context.Context, ref configuration.Ref, key *url.URL) ([]byte, error) { // FTL__[]_ where and are base64 encoded. @@ -37,7 +47,7 @@ func (e Envar[R]) Delete(ctx context.Context, ref configuration.Ref) error { func (e Envar[R]) Store(ctx context.Context, ref configuration.Ref, value []byte) (*url.URL, error) { envar := e.key(ref) fmt.Printf("%s=%s\n", envar, base64.RawURLEncoding.EncodeToString(value)) - return &url.URL{Scheme: "envar", Host: ref.Name}, nil + return &url.URL{Scheme: string(EnvarProviderKey), Host: ref.Name}, nil } func (e Envar[R]) key(ref configuration.Ref) string { diff --git a/internal/configuration/providers/inline_provider.go b/internal/configuration/providers/inline_provider.go index 5f124b89cb..f00320a790 100644 --- a/internal/configuration/providers/inline_provider.go +++ b/internal/configuration/providers/inline_provider.go @@ -9,13 +9,25 @@ import ( "github.com/TBD54566975/ftl/internal/configuration" ) +const InlineProviderKey configuration.ProviderKey = "inline" + // Inline is a configuration provider that stores configuration in its key. type Inline[R configuration.Role] struct{} var _ configuration.SynchronousProvider[configuration.Configuration] = Inline[configuration.Configuration]{} -func (Inline[R]) Role() R { var r R; return r } -func (Inline[R]) Key() string { return "inline" } +func NewInline[R configuration.Role]() Inline[R] { + return Inline[R]{} +} + +func NewInlineFactory[R configuration.Role]() (configuration.ProviderKey, Factory[R]) { + return InlineProviderKey, func(ctx context.Context) (configuration.Provider[R], error) { + return NewInline[R](), nil + } +} + +func (Inline[R]) Role() R { var r R; return r } +func (Inline[R]) Key() configuration.ProviderKey { return InlineProviderKey } func (Inline[R]) Load(ctx context.Context, ref configuration.Ref, key *url.URL) ([]byte, error) { data, err := base64.RawURLEncoding.DecodeString(key.Host) @@ -27,7 +39,7 @@ func (Inline[R]) Load(ctx context.Context, ref configuration.Ref, key *url.URL) func (Inline[R]) Store(ctx context.Context, ref configuration.Ref, value []byte) (*url.URL, error) { b64 := base64.RawURLEncoding.EncodeToString(value) - return &url.URL{Scheme: "inline", Host: b64}, nil + return &url.URL{Scheme: string(InlineProviderKey), Host: b64}, nil } func (Inline[R]) Delete(ctx context.Context, ref configuration.Ref) error { diff --git a/internal/configuration/providers/keychain_provider.go b/internal/configuration/providers/keychain_provider.go index 97722e67c7..c8b117e015 100644 --- a/internal/configuration/providers/keychain_provider.go +++ b/internal/configuration/providers/keychain_provider.go @@ -12,12 +12,24 @@ import ( "github.com/TBD54566975/ftl/internal/configuration" ) +const KeychainProviderKey configuration.ProviderKey = "keychain" + type Keychain struct{} var _ configuration.SynchronousProvider[configuration.Secrets] = Keychain{} -func (Keychain) Role() configuration.Secrets { return configuration.Secrets{} } -func (k Keychain) Key() string { return "keychain" } +func NewKeychain() Keychain { + return Keychain{} +} + +func NewKeychainFactory() (configuration.ProviderKey, Factory[configuration.Secrets]) { + return KeychainProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Secrets], error) { + return NewKeychain(), nil + } +} + +func (Keychain) Role() configuration.Secrets { return configuration.Secrets{} } +func (k Keychain) Key() configuration.ProviderKey { return KeychainProviderKey } func (k Keychain) Load(ctx context.Context, ref configuration.Ref, key *url.URL) ([]byte, error) { value, err := keyring.Get(k.serviceName(ref), key.Host) @@ -35,7 +47,7 @@ func (k Keychain) Store(ctx context.Context, ref configuration.Ref, value []byte if err != nil { return nil, fmt.Errorf("failed to set keychain entry for %q: %w", ref, err) } - return &url.URL{Scheme: "keychain", Host: ref.Name}, nil + return &url.URL{Scheme: string(KeychainProviderKey), Host: ref.Name}, nil } func (k Keychain) Delete(ctx context.Context, ref configuration.Ref) error { diff --git a/internal/configuration/providers/providerstest/manual_sync_utils.go b/internal/configuration/providers/providerstest/manual_sync_utils.go index aa865115b9..4a66e1333f 100644 --- a/internal/configuration/providers/providerstest/manual_sync_utils.go +++ b/internal/configuration/providers/providerstest/manual_sync_utils.go @@ -51,7 +51,7 @@ func (a *ManualSyncProvider[R]) Role() R { return a.provider.Role() } -func (a *ManualSyncProvider[R]) Key() string { +func (a *ManualSyncProvider[R]) Key() configuration.ProviderKey { return a.provider.Key() } diff --git a/internal/configuration/providers/registry.go b/internal/configuration/providers/registry.go new file mode 100644 index 0000000000..9a59e366d5 --- /dev/null +++ b/internal/configuration/providers/registry.go @@ -0,0 +1,34 @@ +package providers + +import ( + "context" + "fmt" + + "github.com/TBD54566975/ftl/internal/configuration" +) + +type Factory[R configuration.Role] func(ctx context.Context) (configuration.Provider[R], error) + +// Registry that lazily constructs configuration providers. +type Registry[R configuration.Role] struct { + factories map[configuration.ProviderKey]Factory[R] +} + +func NewRegistry[R configuration.Role]() *Registry[R] { + return &Registry[R]{ + factories: map[configuration.ProviderKey]Factory[R]{}, + } +} + +func (r *Registry[R]) Register(name configuration.ProviderKey, factory Factory[R]) { + r.factories[name] = factory +} + +func (r *Registry[R]) Get(ctx context.Context, name configuration.ProviderKey) (configuration.Provider[R], error) { + factory, ok := r.factories[name] + if !ok { + var role R + return nil, fmt.Errorf("%s: %s provider not found", name, role) + } + return factory(ctx) +} diff --git a/internal/configuration/routers/file_router.go b/internal/configuration/routers/file_router.go new file mode 100644 index 0000000000..529cc89d5a --- /dev/null +++ b/internal/configuration/routers/file_router.go @@ -0,0 +1,127 @@ +package routers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "sort" + + "github.com/alecthomas/types/optional" + + "github.com/TBD54566975/ftl/internal/configuration" +) + +var _ configuration.Router[configuration.Secrets] = (*FileRouter[configuration.Secrets])(nil) + +// FileRouter is a simple JSON-file-based router for configuration. +type FileRouter[R configuration.Role] struct { + path string +} + +func NewFileRouter[R configuration.Role](path string) *FileRouter[R] { + return &FileRouter[R]{path: path} +} + +func (f *FileRouter[R]) Get(ctx context.Context, ref configuration.Ref) (key *url.URL, err error) { + conf, err := f.load() + if err != nil { + return nil, fmt.Errorf("get %s: %w", ref, err) + } + key, ok := conf[ref] + if !ok { + ref.Module = optional.None[string]() + key, ok = conf[ref] + if !ok { + return nil, fmt.Errorf("get %s: %w", ref, configuration.ErrNotFound) + } + } + return key, nil +} + +func (f *FileRouter[R]) List(ctx context.Context) ([]configuration.Entry, error) { + conf, err := f.load() + if err != nil { + return nil, fmt.Errorf("list: %w", err) + } + out := make([]configuration.Entry, 0, len(conf)) + for ref, key := range conf { + out = append(out, configuration.Entry{Ref: ref, Accessor: key}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Ref.String() < out[j].Ref.String() }) + return out, nil +} + +func (f *FileRouter[R]) Role() (role R) { return } + +func (f *FileRouter[R]) Set(ctx context.Context, ref configuration.Ref, key *url.URL) error { + conf, err := f.load() + if err != nil { + return fmt.Errorf("set %s: %w", ref, err) + } + conf[ref] = key + if err = f.save(conf); err != nil { + return fmt.Errorf("set %s: %w", ref, err) + } + return nil +} + +func (f *FileRouter[R]) Unset(ctx context.Context, ref configuration.Ref) error { + conf, err := f.load() + if err != nil { + return fmt.Errorf("unset %s: %w", ref, err) + } + delete(conf, ref) + if err = f.save(conf); err != nil { + return fmt.Errorf("unset %s: %w", ref, err) + } + return nil +} + +func (f *FileRouter[R]) load() (map[configuration.Ref]*url.URL, error) { + r, err := os.Open(f.path) + if errors.Is(err, os.ErrNotExist) { + return map[configuration.Ref]*url.URL{}, nil + } + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + serialisable := map[string]string{} + if err = dec.Decode(&serialisable); err != nil { + return nil, fmt.Errorf("failed to decode %s: %w", f.path, err) + } + out := map[configuration.Ref]*url.URL{} + for refStr, keyStr := range serialisable { + ref, err := configuration.ParseRef(refStr) + if err != nil { + return nil, fmt.Errorf("failed to parse ref %s: %w", refStr, err) + } + key, err := url.Parse(keyStr) + if err != nil { + return nil, fmt.Errorf("failed to parse key %s: %w", keyStr, err) + } + out[ref] = key + } + return out, nil +} + +func (f *FileRouter[R]) save(data map[configuration.Ref]*url.URL) error { + w, err := os.Create(f.path) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + serialisable := map[string]string{} + for ref, key := range data { + serialisable[ref.String()] = key.String() + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err = enc.Encode(serialisable); err != nil { + return fmt.Errorf("failed to encode %s: %w", f.path, err) + } + return nil +} diff --git a/internal/configuration/routers/file_router_test.go b/internal/configuration/routers/file_router_test.go new file mode 100644 index 0000000000..251400b573 --- /dev/null +++ b/internal/configuration/routers/file_router_test.go @@ -0,0 +1,52 @@ +package routers + +import ( + "context" + "net/url" + "path/filepath" + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/alecthomas/types/must" + "github.com/alecthomas/types/optional" + + "github.com/TBD54566975/ftl/internal/configuration" +) + +func TestFileRouter(t *testing.T) { + dir := t.TempDir() + router := NewFileRouter[configuration.Secrets](filepath.Join(dir, "secrets.json")) + ctx := context.Background() + + ref1 := configuration.Ref{Module: optional.Some[string]("foo"), Name: "bar"} + url1 := must.Get(url.Parse("http://example.com")) + ref2 := configuration.Ref{Module: optional.Some[string]("foo"), Name: "baz"} + url2 := must.Get(url.Parse("http://example2.com")) + + err := router.Set(ctx, ref1, url1) + assert.NoError(t, err) + err = router.Set(ctx, ref2, url2) + assert.NoError(t, err) + + entries, err := router.List(ctx) + assert.NoError(t, err) + assert.Equal(t, []configuration.Entry{ + {Ref: ref1, Accessor: url1}, + {Ref: ref2, Accessor: url2}, + }, entries) + + key, err := router.Get(ctx, ref1) + assert.NoError(t, err) + assert.Equal(t, url1, key) + + err = router.Unset(ctx, ref1) + assert.NoError(t, err) + + entries, err = router.List(ctx) + assert.NoError(t, err) + assert.Equal(t, []configuration.Entry{ + {Ref: ref2, Accessor: url2}, + }, entries) + + assert.Equal(t, configuration.Secrets{}, router.Role()) +} diff --git a/internal/profiles/internal/persistence.go b/internal/profiles/internal/persistence.go new file mode 100644 index 0000000000..784dc6b80f --- /dev/null +++ b/internal/profiles/internal/persistence.go @@ -0,0 +1,199 @@ +// Package internal manages the persistent profile configuration of the FTL CLI. +// +// Layout will be something like: +// +// .ftl-project/ +// project.json +// profiles/ +// / +// profile.json +// [secrets.json] +// [config.json] +// +// See the [design document] for more information. +// +// [design document]: https://hackmd.io/@ftl/Sy2GtZKnR +package internal + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/TBD54566975/ftl/internal/configuration" + "github.com/TBD54566975/ftl/internal/slices" +) + +type ProfileType string + +const ( + ProfileTypeLocal ProfileType = "local" + ProfileTypeRemote ProfileType = "remote" +) + +type Profile struct { + Name string `json:"name"` + Endpoint string `json:"endpoint"` + Type ProfileType `json:"type"` + SecretsProvider configuration.ProviderKey `json:"secrets-provider"` + ConfigProvider configuration.ProviderKey `json:"config-provider"` +} + +func (p *Profile) EndpointURL() (*url.URL, error) { + u, err := url.Parse(p.Endpoint) + if err != nil { + return nil, fmt.Errorf("profile endpoint: %w", err) + } + return u, nil +} + +type Project struct { + Realm string `json:"realm"` + FTLMinVersion string `json:"ftl-min-version,omitempty"` + ModuleRoots []string `json:"module-roots,omitempty"` + NoGit bool `json:"no-git,omitempty"` + DefaultProfile string `json:"default-profile,omitempty"` + + Root string `json:"-"` +} + +// Profiles returns the names of all profiles in the project. +func (p Project) Profiles() ([]string, error) { + profileDir := filepath.Join(p.Root, ".ftl-project", "profiles") + profiles, err := filepath.Glob(filepath.Join(profileDir, "*", "profile.json")) + if err != nil { + return nil, fmt.Errorf("profiles: %s: %w", profileDir, err) + } + return slices.Map(profiles, func(p string) string { return filepath.Base(filepath.Dir(p)) }), nil +} + +func (p Project) LoadProfile(name string) (Profile, error) { + profilePath := filepath.Join(p.Root, ".ftl-project", "profiles", name, "profile.json") + r, err := os.Open(profilePath) + if err != nil { + return Profile{}, fmt.Errorf("open %s: %w", profilePath, err) + } + defer r.Close() //nolint:errcheck + + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + profile := Profile{} + if err = dec.Decode(&profile); err != nil { + return Profile{}, fmt.Errorf("decoding %s: %w", profilePath, err) + } + return profile, nil +} + +// SaveProfile saves a profile to the project. +func (p Project) SaveProfile(profile Profile) error { + profilePath := filepath.Join(p.Root, ".ftl-project", "profiles", profile.Name, "profile.json") + if err := os.MkdirAll(filepath.Dir(profilePath), 0700); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(profilePath), err) + } + + w, err := os.Create(profilePath) + if err != nil { + return fmt.Errorf("create %s: %w", profilePath, err) + } + defer w.Close() //nolint:errcheck + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(profile); err != nil { + return fmt.Errorf("encoding %s: %w", profilePath, err) + } + return nil +} + +func (p *Project) Save() error { + profilePath := filepath.Join(p.Root, ".ftl-project", "project.json") + if err := os.MkdirAll(filepath.Dir(profilePath), 0700); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(profilePath), err) + } + + w, err := os.Create(profilePath) + if err != nil { + return fmt.Errorf("create %s: %w", profilePath, err) + } + defer w.Close() //nolint:errcheck + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(p); err != nil { + return fmt.Errorf("encoding %s: %w", profilePath, err) + } + return nil +} + +// LocalSecretsPath returns the path to the secrets file for the given local profile. +func (p *Project) LocalSecretsPath(profile string) string { + return filepath.Join(p.Root, ".ftl-project", "profiles", profile, "secrets.json") +} + +// LocalConfigPath returns the path to the config file for the given local profile. +func (p *Project) LocalConfigPath(profile string) string { + return filepath.Join(p.Root, ".ftl-project", "profiles", profile, "config.json") +} + +func Init(project Project) error { + if project.Root == "" { + return errors.New("project root is empty") + } + if project.DefaultProfile == "" { + project.DefaultProfile = "local" + } + profilePath := filepath.Join(project.Root, ".ftl-project", "project.json") + if err := os.MkdirAll(filepath.Dir(profilePath), 0700); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(profilePath), err) + } + + w, err := os.Create(profilePath) + if err != nil { + return fmt.Errorf("create %s: %w", profilePath, err) + } + defer w.Close() //nolint:errcheck + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + if err := enc.Encode(project); err != nil { + return fmt.Errorf("encoding %s: %w", profilePath, err) + } + + if err = project.SaveProfile(Profile{ + Name: project.DefaultProfile, + Endpoint: "http://localhost:8892", + Type: ProfileTypeLocal, + SecretsProvider: "inline", + ConfigProvider: "inline", + }); err != nil { + return fmt.Errorf("save profile: %w", err) + } + + return nil +} + +// Load the project configuration from the given root directory. +func Load(root string) (Project, error) { + profilePath := filepath.Join(root, ".ftl-project", "project.json") + r, err := os.Open(profilePath) + if errors.Is(err, os.ErrNotExist) { + return Project{ + Root: root, + }, nil + } else if err != nil { + return Project{}, fmt.Errorf("open %s: %w", profilePath, err) + } + defer r.Close() //nolint:errcheck + + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + project := Project{} + if err = dec.Decode(&project); err != nil { + return Project{}, fmt.Errorf("decoding %s: %w", profilePath, err) + } + project.Root = root + return project, nil +} diff --git a/internal/profiles/profiles.go b/internal/profiles/profiles.go new file mode 100644 index 0000000000..e7d488569b --- /dev/null +++ b/internal/profiles/profiles.go @@ -0,0 +1,107 @@ +package profiles + +import ( + "context" + "fmt" + "net/url" + + "github.com/TBD54566975/ftl/internal/configuration" + "github.com/TBD54566975/ftl/internal/configuration/manager" + "github.com/TBD54566975/ftl/internal/configuration/providers" + "github.com/TBD54566975/ftl/internal/configuration/routers" + "github.com/TBD54566975/ftl/internal/profiles/internal" +) + +type ProjectConfig internal.Project + +type Config struct { + Name string + Endpoint *url.URL +} + +type Profile struct { + shared ProjectConfig + config Config + sm *manager.Manager[configuration.Secrets] + cm *manager.Manager[configuration.Configuration] +} + +// ProjectConfig is static project-wide configuration shared by all profiles. +func (p *Profile) ProjectConfig() ProjectConfig { return p.shared } + +// Config is the static configuration for a Profile. +func (p *Profile) Config() Config { return p.config } + +func (p *Profile) SecretsManager() *manager.Manager[configuration.Secrets] { return p.sm } +func (p *Profile) ConfigurationManager() *manager.Manager[configuration.Configuration] { return p.cm } + +// Init a new project with a default "local" profile. +func Init(project ProjectConfig) error { + err := internal.Init(internal.Project(project)) + if err != nil { + return fmt.Errorf("init project: %w", err) + } + return nil +} + +// Load a profile from the project. +func Load( + ctx context.Context, + secretsRegistry *providers.Registry[configuration.Secrets], + configRegistry *providers.Registry[configuration.Configuration], + root string, + profile string, +) (Profile, error) { + project, err := internal.Load(root) + if err != nil { + return Profile{}, fmt.Errorf("load project: %w", err) + } + prof, err := project.LoadProfile(profile) + if err != nil { + return Profile{}, fmt.Errorf("load profile: %w", err) + } + profileEndpoint, err := prof.EndpointURL() + if err != nil { + return Profile{}, fmt.Errorf("profile endpoint: %w", err) + } + + var sm *manager.Manager[configuration.Secrets] + var cm *manager.Manager[configuration.Configuration] + switch prof.Type { + case internal.ProfileTypeLocal: + sp, err := secretsRegistry.Get(ctx, prof.SecretsProvider) + if err != nil { + return Profile{}, fmt.Errorf("get secrets provider: %w", err) + } + secretsRouter := routers.NewFileRouter[configuration.Secrets](project.LocalSecretsPath(profile)) + sm, err = manager.New[configuration.Secrets](ctx, secretsRouter, []configuration.Provider[configuration.Secrets]{sp}) + if err != nil { + return Profile{}, fmt.Errorf("create secrets manager: %w", err) + } + + cp, err := configRegistry.Get(ctx, prof.ConfigProvider) + if err != nil { + return Profile{}, fmt.Errorf("get config provider: %w", err) + } + configRouter := routers.NewFileRouter[configuration.Configuration](project.LocalConfigPath(profile)) + cm, err = manager.New[configuration.Configuration](ctx, configRouter, []configuration.Provider[configuration.Configuration]{cp}) + if err != nil { + return Profile{}, fmt.Errorf("create configuration manager: %w", err) + } + + case internal.ProfileTypeRemote: + panic("not implemented") + + default: + return Profile{}, fmt.Errorf("%s: unknown profile type: %q", profile, prof.Type) + } + return Profile{ + shared: ProjectConfig(project), + config: Config{ + Name: prof.Name, + Endpoint: profileEndpoint, + }, + sm: sm, + cm: cm, + }, nil +} diff --git a/internal/profiles/profiles_test.go b/internal/profiles/profiles_test.go new file mode 100644 index 0000000000..f09b6a943f --- /dev/null +++ b/internal/profiles/profiles_test.go @@ -0,0 +1,63 @@ +package profiles_test + +import ( + "context" + "net/url" + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/alecthomas/types/must" + + "github.com/TBD54566975/ftl" + "github.com/TBD54566975/ftl/internal/configuration" + "github.com/TBD54566975/ftl/internal/configuration/providers" + "github.com/TBD54566975/ftl/internal/profiles" +) + +func TestProfile(t *testing.T) { + root := t.TempDir() + + ctx := context.Background() + project := profiles.ProjectConfig{ + Root: root, + Realm: "test", + FTLMinVersion: ftl.Version, + ModuleRoots: []string{"."}, + NoGit: true, + } + err := profiles.Init(project) + assert.NoError(t, err) + + sr := providers.NewRegistry[configuration.Secrets]() + sr.Register(providers.NewInlineFactory[configuration.Secrets]()) + cr := providers.NewRegistry[configuration.Configuration]() + cr.Register(providers.NewInlineFactory[configuration.Configuration]()) + + local, err := profiles.Load(ctx, sr, cr, root, "local") + assert.NoError(t, err) + + assert.Equal(t, profiles.Config{ + Name: "local", + Endpoint: must.Get(url.Parse("http://localhost:8892")), + }, local.Config()) + + assert.Equal(t, profiles.ProjectConfig{ + Root: root, + Realm: "test", + FTLMinVersion: ftl.Version, + ModuleRoots: []string{"."}, + NoGit: true, + DefaultProfile: "local", + }, local.ProjectConfig()) + + cm := local.ConfigurationManager() + passwordKey := configuration.NewRef("echo", "password") + err = cm.Set(ctx, providers.InlineProviderKey, passwordKey, "hello") + assert.NoError(t, err) + + var passwordValue string + err = cm.Get(ctx, passwordKey, &passwordValue) + assert.NoError(t, err) + + assert.Equal(t, "hello", passwordValue) +} diff --git a/internal/projectconfig/projectconfig.go b/internal/projectconfig/projectconfig.go index cc5cfdbe63..31eddc6f36 100644 --- a/internal/projectconfig/projectconfig.go +++ b/internal/projectconfig/projectconfig.go @@ -29,14 +29,14 @@ type Config struct { // Path to the config file. Path string `toml:"-"` - Name string `toml:"name"` - Global ConfigAndSecrets `toml:"global"` - Modules map[string]ConfigAndSecrets `toml:"modules"` - ModuleDirs []string `toml:"module-dirs"` - Commands Commands `toml:"commands"` - FTLMinVersion string `toml:"ftl-min-version"` - Hermit bool `toml:"hermit"` - NoGit bool `toml:"no-git"` + Name string `toml:"name,omitempty"` + Global ConfigAndSecrets `toml:"global,omitempty"` + Modules map[string]ConfigAndSecrets `toml:"modules,omitempty"` + ModuleDirs []string `toml:"module-dirs,omitempty"` + Commands Commands `toml:"commands,omitempty"` + FTLMinVersion string `toml:"ftl-min-version,omitempty"` + Hermit bool `toml:"hermit,omitempty"` + NoGit bool `toml:"no-git,omitempty"` } // Root directory of the project.