fix: use dbmate package for running migrations (#546)

...rather than the CLI tool.

Fixes #540

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Bitfile

@@ -43,11 +43,25 @@ CLIENT_IN = frontend/src/**/*
 NODE_MODULES_OUT = frontend/node_modules
 NODE_MODULES_IN = frontend/package{,-lock}.json
 
+SQLC_IN = sqlc.yaml \
+  backend/controller/sql/schema/*.sql \
+  backend/controller/sql/queries.sql
+SQLC_OUT = backend/controller/sql/db.go \
+  %(grep -q copyfrom backend/controller/sql/queries.sql && echo backend/controller/sql/copyfrom.go || true)% \
+  backend/controller/sql/models.go \
+  backend/controller/sql/queries.sql.go
+
 RUNNER_TEMPLATE_ZIP = backend/controller/scaling/localscaling/template.zip
 
 #virtual release:
 #  inputs: %{RELEASE}/ftl %{RELEASE}/ftl-controller %{RELEASE}/ftl-runner
 
+%{SQLC_OUT}: %{SQLC_IN}
+  build:
+    sqlc generate --experimental
+    # sqlc 1.18.0 generates a file with a missing import
+    gosimports -w backend/controller/sql/querier.go
+
 %{RELEASE}:
   build: install -m 0700 -d %{OUT}
 

Procfile

@@ -1,5 +1,5 @@
 controller: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-controller --key C01H5BRT09Y07547SETZ4HWRA09 --bind http://localhost:8892
-regenerate: watchexec -e yaml -e sql -e proto -i "frontend" -i "examples/**" --debounce 1s -- bit protos/xyz/block/ftl/v1/schema/schema.proto protos/xyz/block/ftl/v1/ftlv1connect/ftl.connect.go backend/controller/internal/sql/db.go
+regenerate: watchexec -e yaml -e sql -e proto -i "frontend" -i "examples/**" --debounce 1s -- bit protos/xyz/block/ftl/v1/schema/schema.proto protos/xyz/block/ftl/v1/ftlv1connect/ftl.connect.go backend/controller/sql/db.go
 runner0: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H5BTS6ABP1EHGZSAGJMBV50A --language go --language kotlin --bind http://localhost:8894 --template-dir build/template
 runner1: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H5BTSGKQ8AZ9S22N9N1SM9HV --language go --language kotlin --bind http://localhost:8895 --template-dir build/template
 runner2: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD0H13WX636B70WV7D216G --language go --language kotlin --bind http://localhost:8896 --template-dir build/template

backend/controller/console.go

@@ -14,7 +14,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/common/model"
 	"github.com/TBD54566975/ftl/backend/common/slices"
-	"github.com/TBD54566975/ftl/backend/controller/internal/dal"
+	"github.com/TBD54566975/ftl/backend/controller/dal"
 	"github.com/TBD54566975/ftl/backend/schema"
 	ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1"
 	pbconsole "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/console"

backend/controller/controller.go

@@ -35,7 +35,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/common/rpc/headers"
 	"github.com/TBD54566975/ftl/backend/common/sha256"
 	"github.com/TBD54566975/ftl/backend/common/slices"
-	"github.com/TBD54566975/ftl/backend/controller/internal/dal"
+	"github.com/TBD54566975/ftl/backend/controller/dal"
 	"github.com/TBD54566975/ftl/backend/controller/scaling"
 	"github.com/TBD54566975/ftl/backend/schema"
 	frontend "github.com/TBD54566975/ftl/frontend"

backend/controller/internal/dal/dal.go → backend/controller/dal/dal.go

@@ -25,8 +25,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/common/pubsub"
 	"github.com/TBD54566975/ftl/backend/common/sha256"
 	"github.com/TBD54566975/ftl/backend/common/slices"
-	"github.com/TBD54566975/ftl/backend/controller/internal/sql"
-	"github.com/TBD54566975/ftl/backend/controller/internal/sqltypes"
+	"github.com/TBD54566975/ftl/backend/controller/sql"
 	"github.com/TBD54566975/ftl/backend/schema"
 	ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1"
 	schemapb "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/schema"
@@ -509,7 +508,7 @@ func (d *DAL) UpsertRunner(ctx context.Context, runner Runner) error {
 		return errors.Wrap(err, "failed to JSON encode runner labels")
 	}
 	deploymentID, err := d.db.UpsertRunner(ctx, sql.UpsertRunnerParams{
-		Key:            sqltypes.Key(runner.Key),
+		Key:            sql.Key(runner.Key),
 		Endpoint:       runner.Endpoint,
 		State:          sql.RunnerState(runner.State),
 		DeploymentName: pgDeploymentName,
@@ -541,7 +540,7 @@ func (d *DAL) KillStaleControllers(ctx context.Context, age time.Duration) (int6
 
 // DeregisterRunner deregisters the given runner.
 func (d *DAL) DeregisterRunner(ctx context.Context, key model.RunnerKey) error {
-	count, err := d.db.DeregisterRunner(ctx, sqltypes.Key(key))
+	count, err := d.db.DeregisterRunner(ctx, sql.Key(key))
 	if err != nil {
 		return errors.WithStack(translatePGError(err))
 	}
@@ -861,15 +860,15 @@ func (d *DAL) GetRoutingTable(ctx context.Context, modules []string) (map[string
 }
 
 func (d *DAL) GetRunnerState(ctx context.Context, runnerKey model.RunnerKey) (RunnerState, error) {
-	state, err := d.db.GetRunnerState(ctx, sqltypes.Key(runnerKey))
+	state, err := d.db.GetRunnerState(ctx, sql.Key(runnerKey))
 	if err != nil {
 		return "", errors.WithStack(translatePGError(err))
 	}
 	return RunnerState(state), nil
 }
 
 func (d *DAL) GetRunner(ctx context.Context, runnerKey model.RunnerKey) (Runner, error) {
-	row, err := d.db.GetRunner(ctx, sqltypes.Key(runnerKey))
+	row, err := d.db.GetRunner(ctx, sql.Key(runnerKey))
 	if err != nil {
 		return Runner{}, errors.WithStack(translatePGError(err))
 	}

backend/controller/internal/dal/dal_test.go → backend/controller/dal/dal_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/common/model"
 	"github.com/TBD54566975/ftl/backend/common/sha256"
-	"github.com/TBD54566975/ftl/backend/controller/internal/sql/sqltest"
+	"github.com/TBD54566975/ftl/backend/controller/sql/sqltest"
 	"github.com/TBD54566975/ftl/backend/schema"
 	ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1"
 )

backend/controller/internal/dal/events.go → backend/controller/dal/events.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/common/model"
-	"github.com/TBD54566975/ftl/backend/controller/internal/sql"
+	"github.com/TBD54566975/ftl/backend/controller/sql"
 	"github.com/TBD54566975/ftl/backend/schema"
 )
 

backend/controller/deployment_logs.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/common/model"
-	"github.com/TBD54566975/ftl/backend/controller/internal/dal"
+	"github.com/TBD54566975/ftl/backend/controller/dal"
 )
 
 var _ log.Sink = (*deploymentLogsSink)(nil)

backend/controller/observability.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/common/model"
-	"github.com/TBD54566975/ftl/backend/controller/internal/dal"
+	"github.com/TBD54566975/ftl/backend/controller/dal"
 	"github.com/TBD54566975/ftl/backend/schema"
 	ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1"
 )

backend/controller/databasetesting/devel.go → backend/controller/sql/databasetesting/devel.go

@@ -7,6 +7,8 @@ import (
 	"github.com/alecthomas/errors"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
+
+	"github.com/TBD54566975/ftl/backend/controller/sql"
 )
 
 // CreateForDevel creates and migrates a new database for development or testing.
@@ -46,7 +48,7 @@ func CreateForDevel(ctx context.Context, dsn string, recreate bool) (*pgxpool.Po
 	// PG doesn't support "IF NOT EXISTS" so instead we just ignore any error.
 	_, _ = conn.Exec(ctx, fmt.Sprintf("CREATE DATABASE %q", config.Database))
 
-	err = Migrate(ctx, dsn)
+	err = sql.Migrate(ctx, dsn)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}

backend/controller/sql/migrate.go

@@ -0,0 +1,41 @@
+package sql
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"net/url"
+
+	"github.com/alecthomas/errors"
+	"github.com/amacneil/dbmate/v2/pkg/dbmate"
+	_ "github.com/amacneil/dbmate/v2/pkg/driver/postgres"
+	_ "github.com/jackc/pgx/v5/stdlib" // SQL driver
+
+	"github.com/TBD54566975/ftl/backend/common/log"
+)
+
+//go:embed schema
+var schema embed.FS
+
+// Migrate the database.
+func Migrate(ctx context.Context, dsn string) error {
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return errors.Wrap(err, "invalid DSN")
+	}
+	conn, err := sql.Open("pgx", dsn)
+	if err != nil {
+		return errors.Wrap(err, "failed to connect to database")
+	}
+	defer conn.Close()
+
+	db := dbmate.New(u)
+	db.FS = schema
+	db.Log = log.FromContext(ctx).Scope("migrate").WriterAt(log.Debug)
+	db.MigrationsDir = []string{"schema"}
+	err = db.CreateAndMigrate()
+	if err != nil {
+		return errors.Wrap(err, "failed to create and migrate database")
+	}
+	return nil
+}