feat: start of profile support

This adds the beginnings of a high-level package for initialising a project,
saving and loading profiles. There's also a lower-level API over the storage
mechanism.

backend/controller/admin/admin.go

@@ -13,6 +13,7 @@ import (
 	"github.com/TBD54566975/ftl/go-runtime/encoding"
 	"github.com/TBD54566975/ftl/internal/configuration"
 	"github.com/TBD54566975/ftl/internal/configuration/manager"
+	"github.com/TBD54566975/ftl/internal/configuration/providers"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
@@ -94,17 +95,17 @@ func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1
 	return connect.NewResponse(&ftlv1.GetConfigResponse{Value: vb}), nil
 }
 
-func configProviderKey(p *ftlv1.ConfigProvider) string {
+func configProviderKey(p *ftlv1.ConfigProvider) configuration.ProviderKey {
 	if p == nil {
 		return ""
 	}
 	switch *p {
 	case ftlv1.ConfigProvider_CONFIG_INLINE:
-		return "inline"
+		return providers.InlineProviderKey
 	case ftlv1.ConfigProvider_CONFIG_ENVAR:
-		return "envar"
+		return providers.EnvarProviderKey
 	case ftlv1.ConfigProvider_CONFIG_DB:
-		return "db"
+		return providers.DatabaseConfigProviderKey
 	}
 	return ""
 }
@@ -188,21 +189,21 @@ func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1
 	return connect.NewResponse(&ftlv1.GetSecretResponse{Value: vb}), nil
 }
 
-func secretProviderKey(p *ftlv1.SecretProvider) string {
+func secretProviderKey(p *ftlv1.SecretProvider) configuration.ProviderKey {
 	if p == nil {
 		return ""
 	}
 	switch *p {
 	case ftlv1.SecretProvider_SECRET_INLINE:
-		return "inline"
+		return providers.InlineProviderKey
 	case ftlv1.SecretProvider_SECRET_ENVAR:
-		return "envar"
+		return providers.EnvarProviderKey
 	case ftlv1.SecretProvider_SECRET_KEYCHAIN:
-		return "keychain"
+		return providers.KeychainProviderKey
 	case ftlv1.SecretProvider_SECRET_OP:
-		return "op"
+		return providers.OnePasswordProviderKey
 	case ftlv1.SecretProvider_SECRET_ASM:
-		return "asm"
+		return providers.ASMProviderKey
 	}
 	return ""
 }

internal/configuration/api.go

@@ -103,10 +103,12 @@ type Router[R Role] interface {
 	List(ctx context.Context) ([]Entry, error)
 }
 
+type ProviderKey string
+
 // Provider is a generic interface for storing and retrieving configuration and secrets.
 type Provider[R Role] interface {
 	Role() R
-	Key() string
+	Key() ProviderKey
 
 	// Store a configuration value and return its key.
 	Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error)

internal/configuration/manager/cache.go

@@ -27,7 +27,7 @@ type listProvider interface {
 }
 
 type updateCacheEvent struct {
-	key string
+	key configuration.ProviderKey
 	ref configuration.Ref
 	// value is nil when the value was deleted
 	value optional.Option[[]byte]
@@ -39,7 +39,7 @@ type updateCacheEvent struct {
 // Sync happens periodically.
 // Updates do not go through the cache, but the cache is notified after the update occurs.
 type cache[R configuration.Role] struct {
-	providers map[string]*cacheProvider[R]
+	providers map[configuration.ProviderKey]*cacheProvider[R]
 
 	// list provider is used to determine which providers are expected to have values, and therefore need to be synced
 	listProvider listProvider
@@ -50,7 +50,7 @@ type cache[R configuration.Role] struct {
 }
 
 func newCache[R configuration.Role](ctx context.Context, providers []configuration.AsynchronousProvider[R], listProvider listProvider) *cache[R] {
-	cacheProviders := make(map[string]*cacheProvider[R], len(providers))
+	cacheProviders := make(map[configuration.ProviderKey]*cacheProvider[R], len(providers))
 	for _, provider := range providers {
 		cacheProviders[provider.Key()] = &cacheProvider[R]{
 			provider:   provider,
@@ -73,7 +73,7 @@ func newCache[R configuration.Role](ctx context.Context, providers []configurati
 // load is called by the manager to get a value from the cache
 func (c *cache[R]) load(ref configuration.Ref, key *url.URL) ([]byte, error) {
 	providerKey := ProviderKeyForAccessor(key)
-	provider, ok := c.providers[key.Scheme]
+	provider, ok := c.providers[configuration.ProviderKey(key.Scheme)]
 	if !ok {
 		return nil, fmt.Errorf("no cache provider for key %q", providerKey)
 	}
@@ -103,7 +103,7 @@ func (c *cache[R]) updatedValue(ref configuration.Ref, value []byte, accessor *u
 }
 
 // deletedValue should be called when a value is deleted in the provider
-func (c *cache[R]) deletedValue(ref configuration.Ref, pkey string) {
+func (c *cache[R]) deletedValue(ref configuration.Ref, pkey configuration.ProviderKey) {
 	if _, ok := c.providers[pkey]; !ok {
 		// not syncing this provider
 		return

internal/configuration/manager/manager.go

@@ -19,7 +19,7 @@ import (
 // Manager is a high-level configuration manager that abstracts the details of
 // the Router and Provider interfaces.
 type Manager[R configuration.Role] struct {
-	providers  map[string]configuration.Provider[R]
+	providers  map[configuration.ProviderKey]configuration.Provider[R]
 	router     configuration.Router[R]
 	obfuscator optional.Option[configuration.Obfuscator]
 	cache      *cache[R]
@@ -49,7 +49,7 @@ func NewDefaultConfigurationManagerFromConfig(ctx context.Context, config string
 // New configuration manager.
 func New[R configuration.Role](ctx context.Context, router configuration.Router[R], providers []configuration.Provider[R]) (*Manager[R], error) {
 	m := &Manager[R]{
-		providers: map[string]configuration.Provider[R]{},
+		providers: map[configuration.ProviderKey]configuration.Provider[R]{},
 	}
 	for _, p := range providers {
 		m.providers[p.Key()] = p
@@ -70,8 +70,8 @@ func New[R configuration.Role](ctx context.Context, router configuration.Router[
 	return m, nil
 }
 
-func ProviderKeyForAccessor(accessor *url.URL) string {
-	return accessor.Scheme
+func ProviderKeyForAccessor(accessor *url.URL) configuration.ProviderKey {
+	return configuration.ProviderKey(accessor.Scheme)
 }
 
 // getData returns a data value for a configuration from the active providers.
@@ -136,13 +136,13 @@ func (m *Manager[R]) Get(ctx context.Context, ref configuration.Ref, value any)
 func (m *Manager[R]) availableProviderKeys() []string {
 	keys := make([]string, 0, len(m.providers))
 	for k := range m.providers {
-		keys = append(keys, "--"+k)
+		keys = append(keys, "--"+string(k))
 	}
 	return keys
 }
 
 // Set a configuration value, encoding "value" as JSON before storing it.
-func (m *Manager[R]) Set(ctx context.Context, pkey string, ref configuration.Ref, value any) error {
+func (m *Manager[R]) Set(ctx context.Context, pkey configuration.ProviderKey, ref configuration.Ref, value any) error {
 	data, err := json.Marshal(value)
 	if err != nil {
 		return err
@@ -151,7 +151,7 @@ func (m *Manager[R]) Set(ctx context.Context, pkey string, ref configuration.Ref
 }
 
 // SetJSON sets a configuration value using raw JSON data.
-func (m *Manager[R]) SetJSON(ctx context.Context, pkey string, ref configuration.Ref, value json.RawMessage) error {
+func (m *Manager[R]) SetJSON(ctx context.Context, pkey configuration.ProviderKey, ref configuration.Ref, value json.RawMessage) error {
 	if err := checkJSON(value); err != nil {
 		return fmt.Errorf("invalid value for %s, must be JSON: %w", m.router.Role(), err)
 	}
@@ -211,7 +211,7 @@ func (m *Manager[R]) MapForModule(ctx context.Context, module string) (map[strin
 }
 
 // Unset a configuration value in all providers.
-func (m *Manager[R]) Unset(ctx context.Context, pkey string, ref configuration.Ref) error {
+func (m *Manager[R]) Unset(ctx context.Context, pkey configuration.ProviderKey, ref configuration.Ref) error {
 	provider, ok := m.providers[pkey]
 	if !ok {
 		pkeys := strings.Join(m.availableProviderKeys(), ", ")

internal/configuration/manager/manager_test.go

@@ -35,7 +35,7 @@ func TestManager(t *testing.T) {
 				kcp,
 			})
 		assert.NoError(t, err)
-		testManager(t, ctx, cf, "keychain", "FTL_SECRET_YmF6", []configuration.Entry{
+		testManager(t, ctx, cf, providers.KeychainProviderKey, "FTL_SECRET_YmF6", []configuration.Entry{
 			{Ref: configuration.Ref{Name: "baz"}, Accessor: URL("envar://baz")},
 			{Ref: configuration.Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")},
 			{Ref: configuration.Ref{Name: "mutable"}, Accessor: URL("keychain://mutable")},
@@ -49,7 +49,7 @@ func TestManager(t *testing.T) {
 				providers.Inline[configuration.Configuration]{},
 			})
 		assert.NoError(t, err)
-		testManager(t, ctx, cf, "inline", "FTL_CONFIG_YmF6", []configuration.Entry{
+		testManager(t, ctx, cf, providers.InlineProviderKey, "FTL_CONFIG_YmF6", []configuration.Entry{
 			{Ref: configuration.Ref{Name: "baz"}, Accessor: URL("envar://baz")},
 			{Ref: configuration.Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")},
 			{Ref: configuration.Ref{Name: "mutable"}, Accessor: URL("inline://ImhlbGxvIg")},
@@ -119,7 +119,7 @@ func testManager[R configuration.Role](
 	t *testing.T,
 	ctx context.Context,
 	cf *Manager[R],
-	providerKey string,
+	providerKey configuration.ProviderKey,
 	envarName string,
 	expectedListing []configuration.Entry,
 ) {

internal/configuration/providers/1password_provider.go

@@ -18,17 +18,33 @@ import (
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
+const OnePasswordProviderKey configuration.ProviderKey = "op"
+
 // OnePassword is a configuration provider that reads passwords from
 // 1Password vaults via the "op" command line tool.
 type OnePassword struct {
 	Vault       string
 	ProjectName string
 }
 
+func NewOnePassword(vault string, projectName string) OnePassword {
+	return OnePassword{
+		Vault:       vault,
+		ProjectName: projectName,
+	}
+}
+
+func NewOnePasswordFactory(vault string, projectName string) (configuration.ProviderKey, Factory[configuration.Secrets]) {
+	return OnePasswordProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Secrets], error) {
+		return NewOnePassword(vault, projectName), nil
+	}
+}
+
+var _ configuration.Provider[configuration.Secrets] = OnePassword{}
 var _ configuration.AsynchronousProvider[configuration.Secrets] = OnePassword{}
 
-func (OnePassword) Role() configuration.Secrets { return configuration.Secrets{} }
-func (o OnePassword) Key() string               { return "op" }
+func (OnePassword) Role() configuration.Secrets      { return configuration.Secrets{} }
+func (o OnePassword) Key() configuration.ProviderKey { return OnePasswordProviderKey }
 func (o OnePassword) Delete(ctx context.Context, ref configuration.Ref) error {
 	return nil
 }
@@ -110,7 +126,7 @@ func (o OnePassword) Store(ctx context.Context, ref configuration.Ref, value []b
 		return nil, fmt.Errorf("vault name %q contains invalid characters. a-z A-Z 0-9 _ . - are valid", o.Vault)
 	}
 
-	url := &url.URL{Scheme: "op", Host: o.Vault}
+	url := &url.URL{Scheme: string(OnePasswordProviderKey), Host: o.Vault}
 
 	// make sure item exists
 	_, err := o.getItem(ctx, o.Vault)

internal/configuration/providers/asm.go

@@ -18,6 +18,8 @@ import (
 	"github.com/TBD54566975/ftl/internal/rpc"
 )
 
+const ASMProviderKey configuration.ProviderKey = "asm"
+
 type asmClient interface {
 	name() string
 	syncInterval() time.Duration
@@ -37,6 +39,12 @@ type ASM struct {
 
 var _ configuration.AsynchronousProvider[configuration.Secrets] = &ASM{}
 
+func NewASMFactory(secretsClient *secretsmanager.Client, advertise *url.URL, leaser leases.Leaser) (configuration.ProviderKey, Factory[configuration.Secrets]) {
+	return ASMProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Secrets], error) {
+		return NewASM(ctx, secretsClient, advertise, leaser), nil
+	}
+}
+
 func NewASM(ctx context.Context, secretsClient *secretsmanager.Client, advertise *url.URL, leaser leases.Leaser) *ASM {
 	return newASMForTesting(ctx, secretsClient, advertise, leaser, optional.None[asmClient]())
 }
@@ -58,7 +66,7 @@ func newASMForTesting(ctx context.Context, secretsClient *secretsmanager.Client,
 	coordinator := leader.NewCoordinator[asmClient](
 		ctx,
 		advertise,
-		leases.SystemKey("asm"),
+		leases.SystemKey(string(ASMProviderKey)),
 		leaser,
 		time.Second*10,
 		leaderFactory,
@@ -71,7 +79,7 @@ func newASMForTesting(ctx context.Context, secretsClient *secretsmanager.Client,
 
 func asmURLForRef(ref configuration.Ref) *url.URL {
 	return &url.URL{
-		Scheme: "asm",
+		Scheme: string(ASMProviderKey),
 		Host:   ref.String(),
 	}
 }
@@ -80,8 +88,8 @@ func (ASM) Role() configuration.Secrets {
 	return configuration.Secrets{}
 }
 
-func (ASM) Key() string {
-	return "asm"
+func (*ASM) Key() configuration.ProviderKey {
+	return ASMProviderKey
 }
 
 func (a *ASM) SyncInterval() time.Duration {

internal/configuration/providers/db_config_provider.go

@@ -11,6 +11,8 @@ import (
 	"github.com/TBD54566975/ftl/internal/configuration"
 )
 
+const DatabaseConfigProviderKey configuration.ProviderKey = "db"
+
 // DatabaseConfig is a configuration provider that stores configuration in its key.
 type DatabaseConfig struct {
 	dal DatabaseConfigDAL
@@ -24,14 +26,20 @@ type DatabaseConfigDAL interface {
 	UnsetModuleConfiguration(ctx context.Context, module optional.Option[string], name string) error
 }
 
+func NewDatabaseConfigFactory(dal DatabaseConfigDAL) (configuration.ProviderKey, Factory[configuration.Configuration]) {
+	return DatabaseConfigProviderKey, func(ctx context.Context) (configuration.Provider[configuration.Configuration], error) {
+		return NewDatabaseConfig(dal), nil
+	}
+}
+
 func NewDatabaseConfig(dal DatabaseConfigDAL) DatabaseConfig {
 	return DatabaseConfig{
 		dal: dal,
 	}
 }
 
 func (DatabaseConfig) Role() configuration.Configuration { return configuration.Configuration{} }
-func (DatabaseConfig) Key() string                       { return "db" }
+func (DatabaseConfig) Key() configuration.ProviderKey    { return DatabaseConfigProviderKey }
 
 func (d DatabaseConfig) Load(ctx context.Context, ref configuration.Ref, key *url.URL) ([]byte, error) {
 	value, err := d.dal.GetModuleConfiguration(ctx, ref.Module, ref.Name)
@@ -46,7 +54,7 @@ func (d DatabaseConfig) Store(ctx context.Context, ref configuration.Ref, value
 	if err != nil {
 		return nil, fmt.Errorf("failed to set configuration: %w", err)
 	}
-	return &url.URL{Scheme: "db"}, nil
+	return &url.URL{Scheme: string(DatabaseConfigProviderKey)}, nil
 }
 
 func (d DatabaseConfig) Delete(ctx context.Context, ref configuration.Ref) error {

internal/configuration/providers/envar_provider.go

@@ -10,14 +10,24 @@ import (
 	"github.com/TBD54566975/ftl/internal/configuration"
 )
 
+const EnvarProviderKey configuration.ProviderKey = "envar"
+
 // Envar is a configuration provider that reads secrets or configuration
 // from environment variables.
 type Envar[R configuration.Role] struct{}
 
 var _ configuration.SynchronousProvider[configuration.Configuration] = Envar[configuration.Configuration]{}
 
-func (Envar[R]) Role() R     { var r R; return r }
-func (Envar[R]) Key() string { return "envar" }
+func NewEnvarFactory[R configuration.Role]() (configuration.ProviderKey, Factory[R]) {
+	return EnvarProviderKey, func(ctx context.Context) (configuration.Provider[R], error) {
+		return NewEnvar[R](), nil
+	}
+}
+
+func NewEnvar[R configuration.Role]() Envar[R] { return Envar[R]{} }
+
+func (Envar[R]) Role() R                        { var r R; return r }
+func (Envar[R]) Key() configuration.ProviderKey { return EnvarProviderKey }
 
 func (e Envar[R]) Load(ctx context.Context, ref configuration.Ref, key *url.URL) ([]byte, error) {
 	// FTL_<type>_[<module>]_<name> where <module> and <name> are base64 encoded.
@@ -37,7 +47,7 @@ func (e Envar[R]) Delete(ctx context.Context, ref configuration.Ref) error {
 func (e Envar[R]) Store(ctx context.Context, ref configuration.Ref, value []byte) (*url.URL, error) {
 	envar := e.key(ref)
 	fmt.Printf("%s=%s\n", envar, base64.RawURLEncoding.EncodeToString(value))
-	return &url.URL{Scheme: "envar", Host: ref.Name}, nil
+	return &url.URL{Scheme: string(EnvarProviderKey), Host: ref.Name}, nil
 }
 
 func (e Envar[R]) key(ref configuration.Ref) string {