feat: use secrets instead of env vars for DSNs (#1483)

Fixes #1427

database/ftl-project.toml was written using:
```
ftl secret set --inline database.FTL_DSN_DATABASE_TESTDB --config integration/testdata/go/database/ftl-project.toml
```

This PR only works thanks to this earlier PR:
#1472

backend/controller/controller.go

@@ -668,7 +668,7 @@ func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftl
 	if err != nil {
 		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get secrets: %w", err))
 	}
-	databases, err := modulecontext.DatabasesFromEnvironment(ctx, name)
+	databases, err := modulecontext.DatabasesFromSecrets(ctx, name, secrets)
 	if err != nil {
 		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get databases: %w", err))
 	}

go-runtime/ftl/ftltest/ftltest.go

@@ -37,15 +37,10 @@ func Context(options ...Option) context.Context {
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	name := ftl.Module()
 
-	databases, err := modulecontext.DatabasesFromEnvironment(ctx, name)
-	if err != nil {
-		panic(fmt.Sprintf("error setting up module context from environment: %v", err))
-	}
-
 	state := &OptionsState{
 		configs:   make(map[string][]byte),
 		secrets:   make(map[string][]byte),
-		databases: databases,
+		databases: make(map[string]modulecontext.Database),
 		mockVerbs: make(map[schema.RefKey]modulecontext.Verb),
 	}
 	for _, option := range options {
@@ -183,7 +178,7 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) Option {
 // )
 func WithDatabase(dbHandle ftl.Database) Option {
 	return func(ctx context.Context, state *OptionsState) error {
-		originalDSN, err := modulecontext.GetDSNFromEnvar(ftl.Module(), dbHandle.Name)
+		originalDSN, err := modulecontext.GetDSNFromSecret(ftl.Module(), dbHandle.Name, state.secrets)
 		if err != nil {
 			return err
 		}

integration/actions_test.go

@@ -26,7 +26,6 @@ import (
 	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema"
 	ftlexec "github.com/TBD54566975/ftl/internal/exec"
 	"github.com/TBD54566975/ftl/internal/log"
-	"github.com/TBD54566975/ftl/internal/modulecontext"
 	"github.com/TBD54566975/scaffolder"
 )
 
@@ -271,10 +270,6 @@ func createDBAction(module, dbName string, isTest bool) action {
 }
 
 func createDB(t testing.TB, module, dbName string, isTestDb bool) {
-	// envars do not include test suffix
-	t.Setenv(modulecontext.DSNEnvarName(module, dbName),
-		fmt.Sprintf("postgres://postgres:secret@localhost:54320/%s?sslmode=disable", dbName))
-
 	// insert test suffix if needed when actually setting up db
 	if isTestDb {
 		dbName += "_test"

integration/harness_test.go

@@ -41,14 +41,18 @@ func infof(format string, args ...any) {
 var buildOnce sync.Once
 
 // run an integration test.
-func run(t *testing.T, actions ...action) {
+func run(t *testing.T, ftlConfigPath string, actions ...action) {
 	tmpDir := t.TempDir()
 
 	cwd, err := os.Getwd()
 	assert.NoError(t, err)
 
 	rootDir := internal.GitRoot("")
 
+	if ftlConfigPath != "" {
+		t.Setenv("FTL_CONFIG", filepath.Join(tmpDir, ftlConfigPath))
+	}
+
 	// Build FTL binary
 	logger := log.Configure(&logWriter{logger: t}, log.Config{Level: log.Debug})
 	ctx := log.ContextWithLogger(context.Background(), logger)

integration/integration_test.go

@@ -30,7 +30,7 @@ func TestCron(t *testing.T) {
 
 	t.Cleanup(func() { _ = os.Remove(tmpFile) })
 
-	run(t,
+	run(t, "",
 		copyModule("cron"),
 		deploy("cron"),
 		func(t testing.TB, ic testContext) error {
@@ -41,7 +41,7 @@ func TestCron(t *testing.T) {
 }
 
 func TestLifecycle(t *testing.T) {
-	run(t,
+	run(t, "",
 		exec("ftl", "init", "go", ".", "echo"),
 		deploy("echo"),
 		call("echo", "echo", obj{"name": "Bob"}, func(response obj) error {
@@ -54,7 +54,7 @@ func TestLifecycle(t *testing.T) {
 }
 
 func TestInterModuleCall(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("echo"),
 		copyModule("time"),
 		deploy("time"),
@@ -73,7 +73,7 @@ func TestInterModuleCall(t *testing.T) {
 }
 
 func TestNonExportedDecls(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("time"),
 		deploy("time"),
 		copyModule("echo"),
@@ -84,10 +84,10 @@ func TestNonExportedDecls(t *testing.T) {
 }
 
 func TestDatabase(t *testing.T) {
-	createDB(t, "database", "testdb", false)
-	run(t,
+	run(t, "database/ftl-project.toml",
 		// deploy real module against "testdb"
 		copyModule("database"),
+		createDBAction("database", "testdb", false),
 		deploy("database"),
 		call("database", "insert", obj{"data": "hello"}, func(response obj) error { return nil }),
 		queryRow("testdb", "SELECT data FROM requests", "hello"),
@@ -100,7 +100,7 @@ func TestDatabase(t *testing.T) {
 }
 
 func TestSchemaGenerate(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyDir("../schema-generate", "schema-generate"),
 		mkdir("build/schema-generate"),
 		exec("ftl", "schema", "generate", "schema-generate", "build/schema-generate"),
@@ -109,7 +109,7 @@ func TestSchemaGenerate(t *testing.T) {
 }
 
 func TestHttpEncodeOmitempty(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("omitempty"),
 		deploy("omitempty"),
 		httpCall(http.MethodGet, "/get", jsonData(t, obj{}), func(resp *httpResponse) error {
@@ -124,7 +124,7 @@ func TestHttpEncodeOmitempty(t *testing.T) {
 }
 
 func TestHttpIngress(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("httpingress"),
 		deploy("httpingress"),
 		httpCall(http.MethodGet, "/users/123/posts/456", jsonData(t, obj{}), func(resp *httpResponse) error {
@@ -243,14 +243,14 @@ func TestHttpIngress(t *testing.T) {
 }
 
 func TestRuntimeReflection(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("runtimereflection"),
 		testModule("runtimereflection"),
 	)
 }
 
 func TestModuleUnitTests(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("time"),
 		copyModule("wrapped"),
 		copyModule("verbtypes"),
@@ -261,7 +261,7 @@ func TestModuleUnitTests(t *testing.T) {
 }
 
 func TestLease(t *testing.T) {
-	run(t,
+	run(t, "",
 		copyModule("leases"),
 		deploy("leases"),
 		func(t testing.TB, ic testContext) error {

integration/testdata/go/database/database_test.go

@@ -10,6 +10,7 @@ import (
 
 func TestDatabase(t *testing.T) {
 	ctx := ftltest.Context(
+		ftltest.WithProjectFiles("ftl-project.toml"),
 		ftltest.WithDatabase(db),
 	)
 
@@ -20,6 +21,7 @@ func TestDatabase(t *testing.T) {
 	assert.Equal(t, "unit test 1", list[0])
 
 	ctx = ftltest.Context(
+		ftltest.WithProjectFiles("ftl-project.toml"),
 		ftltest.WithDatabase(db),
 	)
 

integration/testdata/go/database/ftl-project.toml

@@ -0,0 +1,13 @@
+ftl-min-version = ""
+
+[global]
+
+[modules]
+  [modules.database]
+    [modules.database.secrets]
+      FTL_DSN_DATABASE_TESTDB = "inline://InBvc3RncmVzOi8vcG9zdGdyZXM6c2VjcmV0QGxvY2FsaG9zdDo1NDMyMC90ZXN0ZGI_c3NsbW9kZT1kaXNhYmxlIg"
+
+[executables]
+  ftl = ""
+
+[commands]

internal/modulecontext/from_secrets.go

@@ -0,0 +1,55 @@
+package modulecontext
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// DatabasesFromSecrets finds DSNs in environment variables and creates a map of databases.
+//
+// Environment variables should be in the format FTL_POSTGRES_DSN__<MODULENAME>_<DBNAME>
+func DatabasesFromSecrets(ctx context.Context, module string, secrets map[string][]byte) (map[string]Database, error) {
+	databases := map[string]Database{}
+	for sName, maybeDSN := range secrets {
+		if !strings.HasPrefix(sName, "FTL_DSN_") {
+			continue
+		}
+		// FTL_DSN_<MODULE>_<DBNAME>
+		parts := strings.Split(sName, "_")
+		if len(parts) != 4 {
+			return nil, fmt.Errorf("invalid DSN secret key %q should have format FTL_DSN_MODULE_DBNAME", sName)
+		}
+		moduleName := strings.ToLower(parts[2])
+		dbName := strings.ToLower(parts[3])
+		if !strings.EqualFold(moduleName, module) {
+			continue
+		}
+		dsnStr := string(maybeDSN)
+		dsn := dsnStr[1 : len(dsnStr)-1] // chop leading + trailing quotes
+		db, err := NewDatabase(DBTypePostgres, dsn)
+		if err != nil {
+			return nil, fmt.Errorf("could not create database %q with DSN %q: %w", dbName, maybeDSN, err)
+		}
+		databases[dbName] = db
+	}
+	return databases, nil
+}
+
+// DSNSecretKey returns the key for the secret that is expected to hold the DSN for a database.
+//
+// The format is FTL_DSN_<MODULE>_<DBNAME>
+func DSNSecretKey(module, name string) string {
+	return fmt.Sprintf("FTL_DSN_%s_%s", strings.ToUpper(module), strings.ToUpper(name))
+}
+
+// GetDSNFromSecret returns the DSN for a database from the relevant secret
+func GetDSNFromSecret(module, name string, secrets map[string][]byte) (string, error) {
+	key := DSNSecretKey(module, name)
+	dsn, ok := secrets[key]
+	if !ok {
+		return "", fmt.Errorf("secrets map %v is missing DSN with key %q", secrets, key)
+	}
+	dsnStr := string(dsn)
+	return dsnStr[1 : len(dsnStr)-1], nil // chop leading + trailing quotes
+}

internal/modulecontext/from_environment_test.go → internal/modulecontext/from_secrets_test.go

@@ -9,11 +9,13 @@ import (
 	"github.com/alecthomas/assert/v2"
 )
 
-func TestFromEnvironment(t *testing.T) {
+func TestFromSecrets(t *testing.T) {
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	t.Setenv(DSNEnvarName("echo", "echo"), "postgres://echo:echo@localhost:5432/echo")
-	databases, err := DatabasesFromEnvironment(ctx, "echo")
+	secrets := map[string][]byte{
+		"FTL_DSN_ECHO_ECHO": []byte("\"postgres://echo:echo@localhost:5432/echo\""),
+	}
+	databases, err := DatabasesFromSecrets(ctx, "echo", secrets)
 	assert.NoError(t, err)
 
 	response := NewBuilder("echo").AddDatabases(databases).Build().ToProto()