feat: Add ftltest.WithProjectFile(…) (#1411)

Adds the ability for module tests to use a project file for configs and
secrets.
This allows them to use the standard project toml file, or a specific
toml file on a per test basis
```
ctx := ftltest.Context(
  ftltest.WithProjectFile("path/to/ftl-project.toml"),
  ...
)
```

Other changes:
- modulecontext has been moved to `/internal` as it is not meant to be
part of ftl's public API
- modulecontext no longer uses the `configuration` package. The steps to
create a modulecontext from a project file are now:
- create a manager with the
path:`cf.NewDefaultConfigurationManagerFromConfig([]string{ path })`
- `ConfigFromEnvironment()` allows the passed in config to be used if
that is desired
- extract the relevant info and pass it into a module context builder:
`modulecontext.NewBuilder(name).AddConfigs(manager.MapForModule(name))`
- This means a lot of modules have had their go.mod and go.sum files
updated
- `UpdateFromEnvironment` has been split into accessing configuration
directly (see above) and getting databases from the environment and
passing that into a builder like so:
```
databases, err := DatabasesFromEnvironment(ctx, "echo")
...
response := NewBuilder("echo").AddDatabases(databases).Build()
```
- Test for reading in project toml into modulecontext has been moved to
an integration test with the `wrapped` module, which now a test case for
using `ftltest.WithProjectFile(...)`

backend/controller/controller.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"connectrpc.com/connect"
+	cf "github.com/TBD54566975/ftl/common/configuration"
 	"github.com/alecthomas/concurrency"
 	"github.com/alecthomas/kong"
 	"github.com/alecthomas/types/optional"
@@ -40,11 +41,11 @@ import (
 	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema"
 	"github.com/TBD54566975/ftl/backend/schema"
 	frontend "github.com/TBD54566975/ftl/frontend"
-	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
 	"github.com/TBD54566975/ftl/internal/cors"
 	"github.com/TBD54566975/ftl/internal/log"
 	ftlmaps "github.com/TBD54566975/ftl/internal/maps"
 	"github.com/TBD54566975/ftl/internal/model"
+	"github.com/TBD54566975/ftl/internal/modulecontext"
 	"github.com/TBD54566975/ftl/internal/rpc"
 	"github.com/TBD54566975/ftl/internal/rpc/headers"
 	"github.com/TBD54566975/ftl/internal/sha256"
@@ -632,23 +633,30 @@ nextModule:
 
 // GetModuleContext retrieves config, secrets and DSNs for a module.
 func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftlv1.ModuleContextRequest]) (*connect.Response[ftlv1.ModuleContextResponse], error) {
-	// get module schema
-	schemas, err := s.dal.GetActiveDeploymentSchemas(ctx)
+	name := req.Msg.Module
+
+	cm, err := cf.NewDefaultConfigurationManagerFromConfig(ctx, cf.ConfigFromEnvironment())
 	if err != nil {
-		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get active schemas: %w", err))
+		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get configs: %w", err))
 	}
-	module, ok := slices.Find(schemas, func(s *schema.Module) bool { return s.Name == req.Msg.Module })
-	if !ok {
-		return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("module %q not found", req.Msg.Module))
+	configs, err := cm.MapForModule(ctx, name)
+	if err != nil {
+		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get configs: %w", err))
 	}
-	builder, err := modulecontext.NewBuilder(module.Name).UpdateFromEnvironment(ctx)
+	sm, err := cf.NewDefaultSecretsManagerFromConfig(ctx, cf.ConfigFromEnvironment())
 	if err != nil {
-		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get module context: %w", err))
+		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get secrets: %w", err))
 	}
-	response, err := builder.Build().ToProto()
+	secrets, err := sm.MapForModule(ctx, name)
 	if err != nil {
-		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not marshal module context: %w", err))
+		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get secrets: %w", err))
 	}
+	databases, err := modulecontext.DatabasesFromEnvironment(ctx, name)
+	if err != nil {
+		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get databases: %w", err))
+	}
+
+	response := modulecontext.NewBuilder(name).AddConfigs(configs).AddSecrets(secrets).AddDatabases(databases).Build().ToProto()
 	return connect.NewResponse(response), nil
 }
 

buildengine/testdata/projects/alpha/go.mod

@@ -8,10 +8,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

buildengine/testdata/projects/another/go.mod

@@ -8,10 +8,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

buildengine/testdata/projects/other/go.mod

@@ -8,10 +8,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

common/configuration/manager.go

@@ -32,29 +32,25 @@ type Manager[R Role] struct {
 	resolver  Resolver[R]
 }
 
-func configFromEnvironment() []string {
+func ConfigFromEnvironment() []string {
 	if envar, ok := os.LookupEnv("FTL_CONFIG"); ok {
 		return strings.Split(envar, ",")
 	}
 	return nil
 }
 
-// NewDefaultSecretsManagerFromEnvironment creates a new secrets manager from
-// the default ftl-project.toml.
-func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secrets], error) {
-	var cr Resolver[Secrets] = ProjectConfigResolver[Secrets]{Config: configFromEnvironment()}
-	return DefaultSecretsMixin{
-		InlineProvider: InlineProvider[Secrets]{},
-	}.NewSecretsManager(ctx, cr)
+// NewDefaultSecretsManagerFromConfig creates a new secrets manager from
+// the project config found in the config paths.
+func NewDefaultSecretsManagerFromConfig(ctx context.Context, config []string) (*Manager[Secrets], error) {
+	var cr Resolver[Secrets] = ProjectConfigResolver[Secrets]{Config: config}
+	return DefaultSecretsMixin{}.NewSecretsManager(ctx, cr)
 }
 
-// NewDefaultConfigurationManagerFromEnvironment creates a new configuration
-// manager from the default ftl-project.toml.
-func NewDefaultConfigurationManagerFromEnvironment(ctx context.Context) (*Manager[Configuration], error) {
-	cr := ProjectConfigResolver[Configuration]{Config: configFromEnvironment()}
-	return DefaultConfigMixin{
-		InlineProvider: InlineProvider[Configuration]{},
-	}.NewConfigurationManager(ctx, cr)
+// NewDefaultConfigurationManagerFromConfig creates a new configuration manager from
+// the project config found in the config paths.
+func NewDefaultConfigurationManagerFromConfig(ctx context.Context, config []string) (*Manager[Configuration], error) {
+	cr := ProjectConfigResolver[Configuration]{Config: config}
+	return DefaultConfigMixin{}.NewConfigurationManager(ctx, cr)
 }
 
 // New configuration manager.

examples/go/echo/go.mod

@@ -10,10 +10,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

go-runtime/compile/testdata/failing/go.mod

@@ -10,10 +10,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

go-runtime/compile/testdata/one/go.mod

@@ -10,10 +10,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

go-runtime/compile/testdata/two/go.mod

@@ -10,10 +10,7 @@ require (
 	connectrpc.com/connect v1.16.1 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
 	connectrpc.com/otelconnect v0.7.0 // indirect
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/TBD54566975/scaffolder v0.8.0 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
-	github.com/alecthomas/kong v0.9.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/types v0.14.0 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect

go-runtime/ftl/call.go

@@ -11,7 +11,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/go-runtime/encoding"
-	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
+	"github.com/TBD54566975/ftl/internal/modulecontext"
 	"github.com/TBD54566975/ftl/internal/rpc"
 )
 

go-runtime/ftl/config.go

@@ -6,7 +6,7 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
+	"github.com/TBD54566975/ftl/internal/modulecontext"
 )
 
 // ConfigType is a type that can be used as a configuration value.

go-runtime/ftl/config_test.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/alecthomas/assert/v2"
 
-	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
 	"github.com/TBD54566975/ftl/internal/log"
+	"github.com/TBD54566975/ftl/internal/modulecontext"
 )
 
 func TestConfig(t *testing.T) {