refactor: move logic around filtering config for a module into the Co…

…nfigurationManager (#1330)

backend/controller/module_context.go

@@ -13,15 +13,13 @@ import (
 
 func moduleContextToProto(ctx context.Context, module *schema.Module) (*ftlv1.ModuleContextResponse, error) {
 	// configs
-	configManager := cf.ConfigFromContext(ctx)
-	configMap, err := bytesMapFromConfigManager(ctx, configManager, module.Name)
+	configMap, err := cf.ConfigFromContext(ctx).MapForModule(ctx, module.Name)
 	if err != nil {
 		return nil, err
 	}
 
 	// secrets
-	secretsManager := cf.SecretsFromContext(ctx)
-	secretsMap, err := bytesMapFromConfigManager(ctx, secretsManager, module.Name)
+	secretsMap, err := cf.SecretsFromContext(ctx).MapForModule(ctx, module.Name)
 	if err != nil {
 		return nil, err
 	}
@@ -51,36 +49,3 @@ func moduleContextToProto(ctx context.Context, module *schema.Module) (*ftlv1.Mo
 		Databases: dsnProtos,
 	}, nil
 }
-
-func bytesMapFromConfigManager[R cf.Role](ctx context.Context, manager *cf.Manager[R], moduleName string) (map[string][]byte, error) {
-	configList, err := manager.List(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	// module specific values must override global values
-	// put module specific values into moduleConfigMap, then merge with configMap
-	configMap := map[string][]byte{}
-	moduleConfigMap := map[string][]byte{}
-
-	for _, entry := range configList {
-		refModule, isModuleSpecific := entry.Module.Get()
-		if isModuleSpecific && refModule != moduleName {
-			continue
-		}
-		data, err := manager.GetData(ctx, entry.Ref)
-		if err != nil {
-			return nil, err
-		}
-		if !isModuleSpecific {
-			configMap[entry.Ref.Name] = data
-		} else {
-			moduleConfigMap[entry.Ref.Name] = data
-		}
-	}
-
-	for name, data := range moduleConfigMap {
-		configMap[name] = data
-	}
-	return configMap, nil
-}

common/configuration/manager.go

@@ -64,9 +64,9 @@ func (m *Manager[R]) Mutable() error {
 	return fmt.Errorf("no writeable configuration provider available, specify one of %s", strings.Join(writers, ", "))
 }
 
-// GetData returns a data value for a configuration from the active providers.
+// getData returns a data value for a configuration from the active providers.
 // The data can be unmarshalled from JSON.
-func (m *Manager[R]) GetData(ctx context.Context, ref Ref) ([]byte, error) {
+func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) {
 	key, err := m.resolver.Get(ctx, ref)
 	// Try again at the global scope if the value is not found in module scope.
 	if ref.Module.Ok() && errors.Is(err, ErrNotFound) {
@@ -94,36 +94,58 @@ func (m *Manager[R]) GetData(ctx context.Context, ref Ref) ([]byte, error) {
 //
 // "value" must be a pointer to a Go type that can be unmarshalled from JSON.
 func (m *Manager[R]) Get(ctx context.Context, ref Ref, value any) error {
-	data, err := m.GetData(ctx, ref)
+	data, err := m.getData(ctx, ref)
 	if err != nil {
 		return err
 	}
 	return json.Unmarshal(data, value)
 }
 
-// SetData updates the configuration value in the active writing provider as raw bytes.
-//
-// "data" must be bytes that can be unmarshalled into a Go type.
-func (m *Manager[R]) SetData(ctx context.Context, ref Ref, data []byte) error {
+// Set a configuration value.
+func (m *Manager[R]) Set(ctx context.Context, ref Ref, value any) error {
 	if err := m.Mutable(); err != nil {
 		return err
 	}
+	data, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
 	key, err := m.writer.Store(ctx, ref, data)
 	if err != nil {
 		return err
 	}
 	return m.resolver.Set(ctx, ref, key)
 }
 
-// Set a configuration value in the active writing provider.
-//
-// "value" must be a Go type that can be marshalled to JSON.
-func (m *Manager[R]) Set(ctx context.Context, ref Ref, value any) error {
-	data, err := json.Marshal(value)
+// MapForModule combines all configuration values visible to the module. Local
+// values take precedence.
+func (m *Manager[R]) MapForModule(ctx context.Context, module string) (map[string][]byte, error) {
+	entries, err := m.List(ctx)
 	if err != nil {
-		return err
+		return nil, err
+	}
+	combined := map[string][]byte{}
+	locals := map[string][]byte{}
+	for _, entry := range entries {
+		mod, ok := entry.Module.Get()
+		if ok && mod == module {
+			data, err := m.getData(ctx, entry.Ref)
+			if err != nil {
+				return nil, err
+			}
+			locals[entry.Ref.Name] = data
+		} else if !ok {
+			data, err := m.getData(ctx, entry.Ref)
+			if err != nil {
+				return nil, err
+			}
+			combined[entry.Ref.Name] = data
+		}
+	}
+	for k, v := range locals {
+		combined[k] = v
 	}
-	return m.SetData(ctx, ref, data)
+	return combined, nil
 }
 
 // Unset a configuration value in all providers.

go-runtime/modulecontext/modulecontext.go

@@ -2,8 +2,7 @@ package modulecontext
 
 import (
 	"context"
-
-	"github.com/alecthomas/types/optional"
+	"net/url"
 
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 	cf "github.com/TBD54566975/ftl/common/configuration"
@@ -17,11 +16,11 @@ type ModuleContext struct {
 }
 
 func NewFromProto(ctx context.Context, moduleName string, response *ftlv1.ModuleContextResponse) (*ModuleContext, error) {
-	cm, err := newInMemoryConfigManager[cf.Configuration](ctx)
+	cm, err := newInMemoryConfigManager[cf.Configuration](ctx, response.Configs)
 	if err != nil {
 		return nil, err
 	}
-	sm, err := newInMemoryConfigManager[cf.Secrets](ctx)
+	sm, err := newInMemoryConfigManager[cf.Secrets](ctx, response.Secrets)
 	if err != nil {
 		return nil, err
 	}
@@ -31,12 +30,6 @@ func NewFromProto(ctx context.Context, moduleName string, response *ftlv1.Module
 		dbProvider:     NewDBProvider(),
 	}
 
-	if err := addConfigOrSecrets[cf.Configuration](ctx, *moduleCtx.configManager, response.Configs, moduleName); err != nil {
-		return nil, err
-	}
-	if err := addConfigOrSecrets[cf.Secrets](ctx, *moduleCtx.secretsManager, response.Secrets, moduleName); err != nil {
-		return nil, err
-	}
 	for _, entry := range response.Databases {
 		if err = moduleCtx.dbProvider.Add(entry.Name, DBType(entry.Type), entry.Dsn); err != nil {
 			return nil, err
@@ -45,25 +38,31 @@ func NewFromProto(ctx context.Context, moduleName string, response *ftlv1.Module
 	return moduleCtx, nil
 }
 
-func newInMemoryConfigManager[R cf.Role](ctx context.Context) (*cf.Manager[R], error) {
+func newInMemoryConfigManager[R cf.Role](ctx context.Context, config map[string][]byte) (*cf.Manager[R], error) {
 	provider := cf.NewInMemoryProvider[R]()
+	refs := map[cf.Ref]*url.URL{}
+	for name, data := range config {
+		ref := cf.Ref{Name: name}
+		u, err := provider.Store(ctx, ref, data)
+		if err != nil {
+			return nil, err
+		}
+		refs[ref] = u
+	}
 	resolver := cf.NewInMemoryResolver[R]()
+	for ref, u := range refs {
+		err := resolver.Set(ctx, ref, u)
+		if err != nil {
+			return nil, err
+		}
+	}
 	manager, err := cf.New(ctx, resolver, []cf.Provider[R]{provider})
 	if err != nil {
 		return nil, err
 	}
 	return manager, nil
 }
 
-func addConfigOrSecrets[R cf.Role](ctx context.Context, manager cf.Manager[R], valueMap map[string][]byte, moduleName string) error {
-	for name, data := range valueMap {
-		if err := manager.SetData(ctx, cf.Ref{Module: optional.Some(moduleName), Name: name}, data); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // ApplyToContext sets up the context so that configurations, secrets and DSNs can be retreived
 // Each of these components have accessors to get a manager back from the context
 func (m *ModuleContext) ApplyToContext(ctx context.Context) context.Context {