From 1ddb00beba00f29e64dc372f0487d337b9393ba8 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Fri, 3 Nov 2023 20:55:14 +1100 Subject: [PATCH] fix: use dbmate package for running migrations (#546) ...rather than the CLI tool. Fixes #540 --------- Co-authored-by: github-actions[bot] --- Bitfile | 14 +++++++ Procfile | 2 +- backend/controller/console.go | 2 +- backend/controller/controller.go | 2 +- backend/controller/{internal => }/dal/dal.go | 11 +++-- .../controller/{internal => }/dal/dal_test.go | 2 +- .../controller/{internal => }/dal/events.go | 2 +- .../controller/{internal => }/dal/notify.go | 0 backend/controller/databasetesting/migrate.go | 36 ---------------- backend/controller/deployment_logs.go | 2 +- backend/controller/observability.go | 2 +- backend/controller/{internal => }/sql/conn.go | 0 .../{ => sql}/databasetesting/devel.go | 4 +- backend/controller/{internal => }/sql/db.go | 0 backend/controller/sql/migrate.go | 41 +++++++++++++++++++ .../controller/{internal => }/sql/mixins.go | 0 .../controller/{internal => }/sql/models.go | 5 +-- .../controller/{internal => }/sql/querier.go | 9 ++-- .../controller/{internal => }/sql/queries.sql | 0 .../{internal => }/sql/queries.sql.go | 27 ++++++------ .../{internal => }/sql/schema/001_init.sql | 0 .../{internal => }/sql/sqltest/testing.go | 2 +- .../sqltypes/sqltypes.go => sql/types.go} | 2 +- backend/schema/validate.go | 2 +- cmd/ftl-initdb/main.go | 2 +- cmd/ftl/cmd_serve.go | 2 +- examples/go.mod | 2 +- examples/go.sum | 4 +- examples/online-boutique/go.mod | 2 +- examples/online-boutique/go.sum | 4 +- go.mod | 4 +- go.sum | 14 ++++++- sqlc.yaml | 18 ++++---- 33 files changed, 126 insertions(+), 93 deletions(-) rename backend/controller/{internal => }/dal/dal.go (98%) rename backend/controller/{internal => }/dal/dal_test.go (99%) rename backend/controller/{internal => }/dal/events.go (99%) rename backend/controller/{internal => }/dal/notify.go (100%) delete mode 100644 backend/controller/databasetesting/migrate.go rename backend/controller/{internal => }/sql/conn.go (100%) rename backend/controller/{ => sql}/databasetesting/devel.go (94%) rename backend/controller/{internal => }/sql/db.go (100%) create mode 100644 backend/controller/sql/migrate.go rename backend/controller/{internal => }/sql/mixins.go (100%) rename backend/controller/{internal => }/sql/models.go (97%) rename backend/controller/{internal => }/sql/querier.go (92%) rename backend/controller/{internal => }/sql/queries.sql (100%) rename backend/controller/{internal => }/sql/queries.sql.go (97%) rename backend/controller/{internal => }/sql/schema/001_init.sql (100%) rename backend/controller/{internal => }/sql/sqltest/testing.go (93%) rename backend/controller/{internal/sqltypes/sqltypes.go => sql/types.go} (98%) diff --git a/Bitfile b/Bitfile index 9e66c98789..879823e46e 100644 --- a/Bitfile +++ b/Bitfile @@ -43,11 +43,25 @@ CLIENT_IN = frontend/src/**/* NODE_MODULES_OUT = frontend/node_modules NODE_MODULES_IN = frontend/package{,-lock}.json +SQLC_IN = sqlc.yaml \ + backend/controller/sql/schema/*.sql \ + backend/controller/sql/queries.sql +SQLC_OUT = backend/controller/sql/db.go \ + %(grep -q copyfrom backend/controller/sql/queries.sql && echo backend/controller/sql/copyfrom.go || true)% \ + backend/controller/sql/models.go \ + backend/controller/sql/queries.sql.go + RUNNER_TEMPLATE_ZIP = backend/controller/scaling/localscaling/template.zip #virtual release: # inputs: %{RELEASE}/ftl %{RELEASE}/ftl-controller %{RELEASE}/ftl-runner +%{SQLC_OUT}: %{SQLC_IN} + build: + sqlc generate --experimental + # sqlc 1.18.0 generates a file with a missing import + gosimports -w backend/controller/sql/querier.go + %{RELEASE}: build: install -m 0700 -d %{OUT} diff --git a/Procfile b/Procfile index fd1ae81ce5..af92d48d0d 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,5 @@ controller: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-controller --key C01H5BRT09Y07547SETZ4HWRA09 --bind http://localhost:8892 -regenerate: watchexec -e yaml -e sql -e proto -i "frontend" -i "examples/**" --debounce 1s -- bit protos/xyz/block/ftl/v1/schema/schema.proto protos/xyz/block/ftl/v1/ftlv1connect/ftl.connect.go backend/controller/internal/sql/db.go +regenerate: watchexec -e yaml -e sql -e proto -i "frontend" -i "examples/**" --debounce 1s -- bit protos/xyz/block/ftl/v1/schema/schema.proto protos/xyz/block/ftl/v1/ftlv1connect/ftl.connect.go backend/controller/sql/db.go runner0: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H5BTS6ABP1EHGZSAGJMBV50A --language go --language kotlin --bind http://localhost:8894 --template-dir build/template runner1: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H5BTSGKQ8AZ9S22N9N1SM9HV --language go --language kotlin --bind http://localhost:8895 --template-dir build/template runner2: watchexec -r -e go -i "frontend" -i "examples/**" -- ftl-runner --key R01H8DD0H13WX636B70WV7D216G --language go --language kotlin --bind http://localhost:8896 --template-dir build/template diff --git a/backend/controller/console.go b/backend/controller/console.go index 5febf3c7dc..e5eb79a9d4 100644 --- a/backend/controller/console.go +++ b/backend/controller/console.go @@ -14,7 +14,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/common/model" "github.com/TBD54566975/ftl/backend/common/slices" - "github.com/TBD54566975/ftl/backend/controller/internal/dal" + "github.com/TBD54566975/ftl/backend/controller/dal" "github.com/TBD54566975/ftl/backend/schema" ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1" pbconsole "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/console" diff --git a/backend/controller/controller.go b/backend/controller/controller.go index 5be547e4b1..65ccb12041 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -35,7 +35,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/rpc/headers" "github.com/TBD54566975/ftl/backend/common/sha256" "github.com/TBD54566975/ftl/backend/common/slices" - "github.com/TBD54566975/ftl/backend/controller/internal/dal" + "github.com/TBD54566975/ftl/backend/controller/dal" "github.com/TBD54566975/ftl/backend/controller/scaling" "github.com/TBD54566975/ftl/backend/schema" frontend "github.com/TBD54566975/ftl/frontend" diff --git a/backend/controller/internal/dal/dal.go b/backend/controller/dal/dal.go similarity index 98% rename from backend/controller/internal/dal/dal.go rename to backend/controller/dal/dal.go index 1169202b91..f60b3eca38 100644 --- a/backend/controller/internal/dal/dal.go +++ b/backend/controller/dal/dal.go @@ -25,8 +25,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/pubsub" "github.com/TBD54566975/ftl/backend/common/sha256" "github.com/TBD54566975/ftl/backend/common/slices" - "github.com/TBD54566975/ftl/backend/controller/internal/sql" - "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes" + "github.com/TBD54566975/ftl/backend/controller/sql" "github.com/TBD54566975/ftl/backend/schema" ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1" schemapb "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1/schema" @@ -509,7 +508,7 @@ func (d *DAL) UpsertRunner(ctx context.Context, runner Runner) error { return errors.Wrap(err, "failed to JSON encode runner labels") } deploymentID, err := d.db.UpsertRunner(ctx, sql.UpsertRunnerParams{ - Key: sqltypes.Key(runner.Key), + Key: sql.Key(runner.Key), Endpoint: runner.Endpoint, State: sql.RunnerState(runner.State), DeploymentName: pgDeploymentName, @@ -541,7 +540,7 @@ func (d *DAL) KillStaleControllers(ctx context.Context, age time.Duration) (int6 // DeregisterRunner deregisters the given runner. func (d *DAL) DeregisterRunner(ctx context.Context, key model.RunnerKey) error { - count, err := d.db.DeregisterRunner(ctx, sqltypes.Key(key)) + count, err := d.db.DeregisterRunner(ctx, sql.Key(key)) if err != nil { return errors.WithStack(translatePGError(err)) } @@ -861,7 +860,7 @@ func (d *DAL) GetRoutingTable(ctx context.Context, modules []string) (map[string } func (d *DAL) GetRunnerState(ctx context.Context, runnerKey model.RunnerKey) (RunnerState, error) { - state, err := d.db.GetRunnerState(ctx, sqltypes.Key(runnerKey)) + state, err := d.db.GetRunnerState(ctx, sql.Key(runnerKey)) if err != nil { return "", errors.WithStack(translatePGError(err)) } @@ -869,7 +868,7 @@ func (d *DAL) GetRunnerState(ctx context.Context, runnerKey model.RunnerKey) (Ru } func (d *DAL) GetRunner(ctx context.Context, runnerKey model.RunnerKey) (Runner, error) { - row, err := d.db.GetRunner(ctx, sqltypes.Key(runnerKey)) + row, err := d.db.GetRunner(ctx, sql.Key(runnerKey)) if err != nil { return Runner{}, errors.WithStack(translatePGError(err)) } diff --git a/backend/controller/internal/dal/dal_test.go b/backend/controller/dal/dal_test.go similarity index 99% rename from backend/controller/internal/dal/dal_test.go rename to backend/controller/dal/dal_test.go index 6537052c00..c1ec93e112 100644 --- a/backend/controller/internal/dal/dal_test.go +++ b/backend/controller/dal/dal_test.go @@ -15,7 +15,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/common/model" "github.com/TBD54566975/ftl/backend/common/sha256" - "github.com/TBD54566975/ftl/backend/controller/internal/sql/sqltest" + "github.com/TBD54566975/ftl/backend/controller/sql/sqltest" "github.com/TBD54566975/ftl/backend/schema" ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1" ) diff --git a/backend/controller/internal/dal/events.go b/backend/controller/dal/events.go similarity index 99% rename from backend/controller/internal/dal/events.go rename to backend/controller/dal/events.go index d81419307a..67f1cfdd0f 100644 --- a/backend/controller/internal/dal/events.go +++ b/backend/controller/dal/events.go @@ -13,7 +13,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/common/model" - "github.com/TBD54566975/ftl/backend/controller/internal/sql" + "github.com/TBD54566975/ftl/backend/controller/sql" "github.com/TBD54566975/ftl/backend/schema" ) diff --git a/backend/controller/internal/dal/notify.go b/backend/controller/dal/notify.go similarity index 100% rename from backend/controller/internal/dal/notify.go rename to backend/controller/dal/notify.go diff --git a/backend/controller/databasetesting/migrate.go b/backend/controller/databasetesting/migrate.go deleted file mode 100644 index 342b754763..0000000000 --- a/backend/controller/databasetesting/migrate.go +++ /dev/null @@ -1,36 +0,0 @@ -package databasetesting - -import ( - "context" - "database/sql" - "os" - "strings" - - "github.com/alecthomas/errors" - _ "github.com/jackc/pgx/v5/stdlib" // SQL driver - - "github.com/TBD54566975/ftl/backend/common/exec" - "github.com/TBD54566975/ftl/backend/common/log" -) - -// Migrate a database connection to the latest schema using Goose. -func Migrate(ctx context.Context, dsn string) error { - conn, err := sql.Open("pgx", dsn) - if err != nil { - return errors.Wrap(err, "failed to connect to database") - } - defer conn.Close() - output, err := exec.Capture(ctx, ".", "git", "rev-parse", "--show-toplevel") - if err != nil { - return errors.Wrap(err, "failed to find git root") - } - workingDir := strings.TrimSpace(string(output)) - cmd := exec.Command(ctx, log.Debug, workingDir, "dbmate", "--url="+dsn, "--migrations-dir=backend/controller/internal/sql/schema", "up") - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - return errors.Wrap(err, "failed to run migrations") - } - return nil -} diff --git a/backend/controller/deployment_logs.go b/backend/controller/deployment_logs.go index 4f8a7c68b4..dd9bed0bda 100644 --- a/backend/controller/deployment_logs.go +++ b/backend/controller/deployment_logs.go @@ -10,7 +10,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/common/model" - "github.com/TBD54566975/ftl/backend/controller/internal/dal" + "github.com/TBD54566975/ftl/backend/controller/dal" ) var _ log.Sink = (*deploymentLogsSink)(nil) diff --git a/backend/controller/observability.go b/backend/controller/observability.go index db38a4d62b..523139bc52 100644 --- a/backend/controller/observability.go +++ b/backend/controller/observability.go @@ -9,7 +9,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/common/model" - "github.com/TBD54566975/ftl/backend/controller/internal/dal" + "github.com/TBD54566975/ftl/backend/controller/dal" "github.com/TBD54566975/ftl/backend/schema" ftlv1 "github.com/TBD54566975/ftl/protos/xyz/block/ftl/v1" ) diff --git a/backend/controller/internal/sql/conn.go b/backend/controller/sql/conn.go similarity index 100% rename from backend/controller/internal/sql/conn.go rename to backend/controller/sql/conn.go diff --git a/backend/controller/databasetesting/devel.go b/backend/controller/sql/databasetesting/devel.go similarity index 94% rename from backend/controller/databasetesting/devel.go rename to backend/controller/sql/databasetesting/devel.go index 68ae2f1439..b0dc24d722 100644 --- a/backend/controller/databasetesting/devel.go +++ b/backend/controller/sql/databasetesting/devel.go @@ -7,6 +7,8 @@ import ( "github.com/alecthomas/errors" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + + "github.com/TBD54566975/ftl/backend/controller/sql" ) // CreateForDevel creates and migrates a new database for development or testing. @@ -46,7 +48,7 @@ func CreateForDevel(ctx context.Context, dsn string, recreate bool) (*pgxpool.Po // PG doesn't support "IF NOT EXISTS" so instead we just ignore any error. _, _ = conn.Exec(ctx, fmt.Sprintf("CREATE DATABASE %q", config.Database)) - err = Migrate(ctx, dsn) + err = sql.Migrate(ctx, dsn) if err != nil { return nil, errors.WithStack(err) } diff --git a/backend/controller/internal/sql/db.go b/backend/controller/sql/db.go similarity index 100% rename from backend/controller/internal/sql/db.go rename to backend/controller/sql/db.go diff --git a/backend/controller/sql/migrate.go b/backend/controller/sql/migrate.go new file mode 100644 index 0000000000..34d3d87b00 --- /dev/null +++ b/backend/controller/sql/migrate.go @@ -0,0 +1,41 @@ +package sql + +import ( + "context" + "database/sql" + "embed" + "net/url" + + "github.com/alecthomas/errors" + "github.com/amacneil/dbmate/v2/pkg/dbmate" + _ "github.com/amacneil/dbmate/v2/pkg/driver/postgres" + _ "github.com/jackc/pgx/v5/stdlib" // SQL driver + + "github.com/TBD54566975/ftl/backend/common/log" +) + +//go:embed schema +var schema embed.FS + +// Migrate the database. +func Migrate(ctx context.Context, dsn string) error { + u, err := url.Parse(dsn) + if err != nil { + return errors.Wrap(err, "invalid DSN") + } + conn, err := sql.Open("pgx", dsn) + if err != nil { + return errors.Wrap(err, "failed to connect to database") + } + defer conn.Close() + + db := dbmate.New(u) + db.FS = schema + db.Log = log.FromContext(ctx).Scope("migrate").WriterAt(log.Debug) + db.MigrationsDir = []string{"schema"} + err = db.CreateAndMigrate() + if err != nil { + return errors.Wrap(err, "failed to create and migrate database") + } + return nil +} diff --git a/backend/controller/internal/sql/mixins.go b/backend/controller/sql/mixins.go similarity index 100% rename from backend/controller/internal/sql/mixins.go rename to backend/controller/sql/mixins.go diff --git a/backend/controller/internal/sql/models.go b/backend/controller/sql/models.go similarity index 97% rename from backend/controller/internal/sql/models.go rename to backend/controller/sql/models.go index 0fee061e8c..04a344efb8 100644 --- a/backend/controller/internal/sql/models.go +++ b/backend/controller/sql/models.go @@ -11,7 +11,6 @@ import ( "time" "github.com/TBD54566975/ftl/backend/common/model" - "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes" "github.com/alecthomas/types" ) @@ -258,10 +257,10 @@ type Request struct { type Runner struct { ID int64 - Key sqltypes.Key + Key Key Created time.Time LastSeen time.Time - ReservationTimeout sqltypes.NullTime + ReservationTimeout NullTime State RunnerState Endpoint string ModuleName types.Option[string] diff --git a/backend/controller/internal/sql/querier.go b/backend/controller/sql/querier.go similarity index 92% rename from backend/controller/internal/sql/querier.go rename to backend/controller/sql/querier.go index 7dd078ee64..0f5656a439 100644 --- a/backend/controller/internal/sql/querier.go +++ b/backend/controller/sql/querier.go @@ -9,7 +9,6 @@ import ( "time" "github.com/TBD54566975/ftl/backend/common/model" - "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes" "github.com/alecthomas/types" ) @@ -20,7 +19,7 @@ type Querier interface { CreateDeployment(ctx context.Context, name model.DeploymentName, moduleName string, schema []byte) error CreateIngressRequest(ctx context.Context, origin Origin, name string, sourceAddr string) error CreateIngressRoute(ctx context.Context, arg CreateIngressRouteParams) error - DeregisterRunner(ctx context.Context, key sqltypes.Key) (int64, error) + DeregisterRunner(ctx context.Context, key Key) (int64, error) ExpireRunnerReservations(ctx context.Context) (int64, error) GetActiveRunners(ctx context.Context, all bool) ([]GetActiveRunnersRow, error) GetAllIngressRoutes(ctx context.Context, all bool) ([]GetAllIngressRoutesRow, error) @@ -44,10 +43,10 @@ type Querier interface { GetModulesByID(ctx context.Context, ids []int64) ([]Module, error) GetProcessList(ctx context.Context) ([]GetProcessListRow, error) // Retrieve routing information for a runner. - GetRouteForRunner(ctx context.Context, key sqltypes.Key) (GetRouteForRunnerRow, error) + GetRouteForRunner(ctx context.Context, key Key) (GetRouteForRunnerRow, error) GetRoutingTable(ctx context.Context, modules []string) ([]GetRoutingTableRow, error) - GetRunner(ctx context.Context, key sqltypes.Key) (GetRunnerRow, error) - GetRunnerState(ctx context.Context, key sqltypes.Key) (RunnerState, error) + GetRunner(ctx context.Context, key Key) (GetRunnerRow, error) + GetRunnerState(ctx context.Context, key Key) (RunnerState, error) GetRunnersForDeployment(ctx context.Context, name model.DeploymentName) ([]GetRunnersForDeploymentRow, error) InsertCallEvent(ctx context.Context, arg InsertCallEventParams) error InsertDeploymentCreatedEvent(ctx context.Context, arg InsertDeploymentCreatedEventParams) error diff --git a/backend/controller/internal/sql/queries.sql b/backend/controller/sql/queries.sql similarity index 100% rename from backend/controller/internal/sql/queries.sql rename to backend/controller/sql/queries.sql diff --git a/backend/controller/internal/sql/queries.sql.go b/backend/controller/sql/queries.sql.go similarity index 97% rename from backend/controller/internal/sql/queries.sql.go rename to backend/controller/sql/queries.sql.go index e3dc8d536e..cc8309c068 100644 --- a/backend/controller/internal/sql/queries.sql.go +++ b/backend/controller/sql/queries.sql.go @@ -11,7 +11,6 @@ import ( "time" "github.com/TBD54566975/ftl/backend/common/model" - "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes" "github.com/alecthomas/types" ) @@ -105,7 +104,7 @@ SELECT COUNT(*) FROM matches ` -func (q *Queries) DeregisterRunner(ctx context.Context, key sqltypes.Key) (int64, error) { +func (q *Queries) DeregisterRunner(ctx context.Context, key Key) (int64, error) { row := q.db.QueryRow(ctx, deregisterRunner, key) var count int64 err := row.Scan(&count) @@ -150,7 +149,7 @@ ORDER BY r.key ` type GetActiveRunnersRow struct { - RunnerKey sqltypes.Key + RunnerKey Key Endpoint string State RunnerState Labels []byte @@ -647,7 +646,7 @@ WHERE r.state = 'assigned' ` type GetIngressRoutesRow struct { - RunnerKey sqltypes.Key + RunnerKey Key Endpoint string Module string Verb string @@ -722,7 +721,7 @@ type GetProcessListRow struct { MinReplicas int32 DeploymentName model.DeploymentName DeploymentLabels []byte - RunnerKey sqltypes.NullKey + RunnerKey NullKey Endpoint types.Option[string] RunnerLabels []byte } @@ -763,14 +762,14 @@ WHERE r.key = $1 type GetRouteForRunnerRow struct { Endpoint string - RunnerKey sqltypes.Key + RunnerKey Key ModuleName types.Option[string] DeploymentName model.DeploymentName State RunnerState } // Retrieve routing information for a runner. -func (q *Queries) GetRouteForRunner(ctx context.Context, key sqltypes.Key) (GetRouteForRunnerRow, error) { +func (q *Queries) GetRouteForRunner(ctx context.Context, key Key) (GetRouteForRunnerRow, error) { row := q.db.QueryRow(ctx, getRouteForRunner, key) var i GetRouteForRunnerRow err := row.Scan( @@ -794,7 +793,7 @@ WHERE state = 'assigned' type GetRoutingTableRow struct { Endpoint string - RunnerKey sqltypes.Key + RunnerKey Key ModuleName types.Option[string] DeploymentName model.DeploymentName } @@ -840,7 +839,7 @@ WHERE r.key = $1 ` type GetRunnerRow struct { - RunnerKey sqltypes.Key + RunnerKey Key Endpoint string State RunnerState Labels []byte @@ -849,7 +848,7 @@ type GetRunnerRow struct { DeploymentName interface{} } -func (q *Queries) GetRunner(ctx context.Context, key sqltypes.Key) (GetRunnerRow, error) { +func (q *Queries) GetRunner(ctx context.Context, key Key) (GetRunnerRow, error) { row := q.db.QueryRow(ctx, getRunner, key) var i GetRunnerRow err := row.Scan( @@ -870,7 +869,7 @@ FROM runners WHERE key = $1 ` -func (q *Queries) GetRunnerState(ctx context.Context, key sqltypes.Key) (RunnerState, error) { +func (q *Queries) GetRunnerState(ctx context.Context, key Key) (RunnerState, error) { row := q.db.QueryRow(ctx, getRunnerState, key) var state RunnerState err := row.Scan(&state) @@ -887,10 +886,10 @@ WHERE state = 'assigned' type GetRunnersForDeploymentRow struct { ID int64 - Key sqltypes.Key + Key Key Created time.Time LastSeen time.Time - ReservationTimeout sqltypes.NullTime + ReservationTimeout NullTime State RunnerState Endpoint string ModuleName types.Option[string] @@ -1293,7 +1292,7 @@ RETURNING deployment_id ` type UpsertRunnerParams struct { - Key sqltypes.Key + Key Key Endpoint string State RunnerState Labels []byte diff --git a/backend/controller/internal/sql/schema/001_init.sql b/backend/controller/sql/schema/001_init.sql similarity index 100% rename from backend/controller/internal/sql/schema/001_init.sql rename to backend/controller/sql/schema/001_init.sql diff --git a/backend/controller/internal/sql/sqltest/testing.go b/backend/controller/sql/sqltest/testing.go similarity index 93% rename from backend/controller/internal/sql/sqltest/testing.go rename to backend/controller/sql/sqltest/testing.go index 77e2bc2002..c420d353a9 100644 --- a/backend/controller/internal/sql/sqltest/testing.go +++ b/backend/controller/sql/sqltest/testing.go @@ -11,7 +11,7 @@ import ( "github.com/gofrs/flock" "github.com/jackc/pgx/v5/pgxpool" - "github.com/TBD54566975/ftl/backend/controller/databasetesting" + "github.com/TBD54566975/ftl/backend/controller/sql/databasetesting" ) // OpenForTesting opens a database connection for testing, recreating the diff --git a/backend/controller/internal/sqltypes/sqltypes.go b/backend/controller/sql/types.go similarity index 98% rename from backend/controller/internal/sqltypes/sqltypes.go rename to backend/controller/sql/types.go index 31030e6a1d..3195edfb40 100644 --- a/backend/controller/internal/sqltypes/sqltypes.go +++ b/backend/controller/sql/types.go @@ -1,4 +1,4 @@ -package sqltypes +package sql import ( "database/sql/driver" diff --git a/backend/schema/validate.go b/backend/schema/validate.go index 19dffdb474..8ef3d854df 100644 --- a/backend/schema/validate.go +++ b/backend/schema/validate.go @@ -120,7 +120,7 @@ func ValidateModule(module *Module) error { case *Array, *Bool, *DataRef, *Field, *Float, *Int, *Time, *Map, *Module, *Schema, *String, *VerbRef, *MetadataCalls, *MetadataIngress: - case Type, Metadata, Decl: // Union sqltypes. + case Type, Metadata, Decl: // Union sql. } return next() }) diff --git a/cmd/ftl-initdb/main.go b/cmd/ftl-initdb/main.go index 948f1c4c88..4b9a4062ab 100644 --- a/cmd/ftl-initdb/main.go +++ b/cmd/ftl-initdb/main.go @@ -7,7 +7,7 @@ import ( "github.com/alecthomas/kong" "github.com/TBD54566975/ftl/backend/common/log" - "github.com/TBD54566975/ftl/backend/controller/databasetesting" + "github.com/TBD54566975/ftl/backend/controller/sql/databasetesting" ) var cli struct { diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go index d43ce62327..a504f05707 100644 --- a/cmd/ftl/cmd_serve.go +++ b/cmd/ftl/cmd_serve.go @@ -15,8 +15,8 @@ import ( "github.com/TBD54566975/ftl/backend/common/exec" "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/controller" - "github.com/TBD54566975/ftl/backend/controller/databasetesting" "github.com/TBD54566975/ftl/backend/controller/scaling/localscaling" + "github.com/TBD54566975/ftl/backend/controller/sql/databasetesting" ) type serveCmd struct { diff --git a/examples/go.mod b/examples/go.mod index 62f312c422..b0a26358b8 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -19,7 +19,7 @@ require ( github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.4.0 // indirect github.com/iancoleman/strcase v0.2.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect diff --git a/examples/go.sum b/examples/go.sum index 6e2f6a18be..e157cfe391 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -41,8 +41,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= diff --git a/examples/online-boutique/go.mod b/examples/online-boutique/go.mod index c24b8e8610..6f115d2f61 100644 --- a/examples/online-boutique/go.mod +++ b/examples/online-boutique/go.mod @@ -7,7 +7,7 @@ replace github.com/TBD54566975/ftl => ../.. require ( github.com/TBD54566975/ftl v0.0.0-00010101000000-000000000000 github.com/alecthomas/errors v0.4.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.4.0 github.com/hashicorp/golang-lru/v2 v2.0.5 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 ) diff --git a/examples/online-boutique/go.sum b/examples/online-boutique/go.sum index 2744ec7936..0321db7610 100644 --- a/examples/online-boutique/go.sum +++ b/examples/online-boutique/go.sum @@ -41,8 +41,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= diff --git a/go.mod b/go.mod index 23632d1c7d..8542fc5ffb 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,10 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/kong v0.8.1 github.com/alecthomas/kong-toml v0.0.1 + github.com/amacneil/dbmate/v2 v2.7.0 github.com/go-logr/logr v1.2.4 github.com/golang/protobuf v1.5.3 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.4.0 github.com/jackc/pgx/v5 v5.3.1 github.com/jellydator/ttlcache/v3 v3.1.0 github.com/oklog/ulid/v2 v2.1.0 @@ -43,6 +44,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.0 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/stretchr/objx v0.2.0 // indirect github.com/swaggest/refl v1.3.0 // indirect diff --git a/go.sum b/go.sum index 201163f5f3..977581be8f 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/alecthomas/types v0.7.1 h1:zU49e5jp0MS4sVDq0yEX4uX/jX7aVpvwd+Rm632ZpE github.com/alecthomas/types v0.7.1/go.mod h1:t7PnU03TVweFpbPVKaeLtFykjJD8rqiBJ7gfkp6UvLQ= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/amacneil/dbmate/v2 v2.7.0 h1:aaTyfVPxf01KyIx5E0i/OATygrit2hjivyKSX0YSJxI= +github.com/amacneil/dbmate/v2 v2.7.0/go.mod h1:I13evRylGros6OXuJij+oskvr+YMaPvSD9Z7PNucXWA= github.com/bool64/dev v0.2.31 h1:OS57EqYaYe2M/2bw9uhDCIFiZZwywKFS/4qMLN6JUmQ= github.com/bool64/dev v0.2.31/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= @@ -48,6 +50,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -61,8 +65,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -93,8 +97,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= @@ -142,6 +150,8 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3Ifn github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/zalando/go-keyring v0.2.1 h1:MBRN/Z8H4U5wEKXiD67YbDAr5cj/DOStmSga70/2qKc= github.com/zalando/go-keyring v0.2.1/go.mod h1:g63M2PPn0w5vjmEbwAX3ib5I+41zdm4esSETOn9Y6Dw= +github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 h1:qXafrlZL1WsJW5OokjraLLRURHiw0OzKHD/RNdspp4w= +github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQxz6hGoNFBC4nIx+CxZhI3nne5RmIOlT/MXcSD4= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= diff --git a/sqlc.yaml b/sqlc.yaml index fa508220fe..78bd8f741d 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -1,15 +1,15 @@ version: "2" sql: - engine: "postgresql" - queries: "backend/controller/internal/sql/queries.sql" - schema: "backend/controller/internal/sql/schema" + queries: "backend/controller/sql/queries.sql" + schema: "backend/controller/sql/schema" database: uri: postgres://localhost/ftl?sslmode=disable&user=postgres&password=secret gen: go: package: "sql" sql_package: "pgx/v5" - out: "backend/controller/internal/sql" + out: "backend/controller/sql" emit_interface: true query_parameter_limit: 3 overrides: @@ -19,15 +19,19 @@ sql: go_type: "time.Duration" - db_type: "pg_catalog.interval" nullable: true - go_type: "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes.NullDuration" + go_type: + type: "NullDuration" - db_type: "timestamptz" nullable: true - go_type: "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes.NullTime" + go_type: + type: "NullTime" - db_type: "uuid" - go_type: "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes.Key" + go_type: + type: "Key" - db_type: "uuid" nullable: true - go_type: "github.com/TBD54566975/ftl/backend/controller/internal/sqltypes.NullKey" + go_type: + type: "NullKey" - db_type: "pg_catalog.varchar" nullable: true go_type: "github.com/alecthomas/types.Option[string]"