From 574918b92ffb72c5a728e0171c9f14ab60d4a113 Mon Sep 17 00:00:00 2001 From: Safeer Jiwan Date: Wed, 3 Jul 2024 15:28:12 -0700 Subject: [PATCH 01/23] fix: database-backed secret resolver (#1901) Fixes #1887 --- backend/controller/sql/models.go | 8 ++ backend/controller/sql/schema/001_init.sql | 10 +++ cmd/ftl-controller/main.go | 6 +- common/configuration/asm.go | 33 +------- common/configuration/asm_test.go | 12 ++- common/configuration/dal/dal.go | 42 ++++++++++ common/configuration/db_secret_resolver.go | 81 ++++++++++++++++++ .../configuration/db_secret_resolver_test.go | 82 +++++++++++++++++++ common/configuration/sql/models.go | 8 ++ common/configuration/sql/querier.go | 4 + common/configuration/sql/queries.sql | 22 +++++ common/configuration/sql/queries.sql.go | 69 ++++++++++++++++ 12 files changed, 339 insertions(+), 38 deletions(-) create mode 100644 common/configuration/db_secret_resolver.go create mode 100644 common/configuration/db_secret_resolver_test.go diff --git a/backend/controller/sql/models.go b/backend/controller/sql/models.go index 6d2095f7ba..8ab61783d8 100644 --- a/backend/controller/sql/models.go +++ b/backend/controller/sql/models.go @@ -478,6 +478,14 @@ type ModuleConfiguration struct { Value []byte } +type ModuleSecret struct { + ID int64 + CreatedAt time.Time + Module optional.Option[string] + Name string + Url string +} + type Request struct { ID int64 Origin Origin diff --git a/backend/controller/sql/schema/001_init.sql b/backend/controller/sql/schema/001_init.sql index 68b34021b2..fb5c20462c 100644 --- a/backend/controller/sql/schema/001_init.sql +++ b/backend/controller/sql/schema/001_init.sql @@ -518,4 +518,14 @@ CREATE TABLE module_configuration UNIQUE (module, name) ); +CREATE TABLE module_secrets +( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'), + module TEXT, -- If NULL, configuration is global. + name TEXT NOT NULL, + url TEXT NOT NULL, + UNIQUE (module, name) +); + -- migrate:down diff --git a/cmd/ftl-controller/main.go b/cmd/ftl-controller/main.go index 1af8897d20..0e85f7735b 100644 --- a/cmd/ftl-controller/main.go +++ b/cmd/ftl-controller/main.go @@ -63,9 +63,9 @@ func main() { // The FTL controller currently only supports AWS Secrets Manager as a secrets provider. awsConfig, err := config.LoadDefaultConfig(ctx) kctx.FatalIfErrorf(err) - secretsResolver := cf.NewASM(ctx, secretsmanager.NewFromConfig(awsConfig), cli.ControllerConfig.Advertise, dal) - secretsProviders := []cf.Provider[cf.Secrets]{secretsResolver} - sm, err := cf.New[cf.Secrets](ctx, secretsResolver, secretsProviders) + asmSecretProvider := cf.NewASM(ctx, secretsmanager.NewFromConfig(awsConfig), cli.ControllerConfig.Advertise, dal) + dbSecretResolver := cf.NewDBSecretResolver(configDal) + sm, err := cf.New[cf.Secrets](ctx, dbSecretResolver, []cf.Provider[cf.Secrets]{asmSecretProvider}) kctx.FatalIfErrorf(err) ctx = cf.ContextWithSecrets(ctx, sm) diff --git a/common/configuration/asm.go b/common/configuration/asm.go index 02456c1128..e1b7eb741d 100644 --- a/common/configuration/asm.go +++ b/common/configuration/asm.go @@ -2,7 +2,6 @@ package configuration import ( "context" - "fmt" "net/url" "time" @@ -24,18 +23,15 @@ type asmClient interface { delete(ctx context.Context, ref Ref) error } -// ASM implements Router and Provider for AWS Secrets Manager (ASM). +// ASM implements a Provider for AWS Secrets Manager (ASM). // Only supports loading "string" secrets, not binary secrets. // -// The router does a direct/proxy map from a Ref to a URL, module.name <-> asm://module.name and does not access ASM at all. -// // One controller is elected as the leader and is responsible for syncing the cache of secrets from ASM (see asmLeader). // Others get secrets from the leader via AdminService (see asmFollower). type ASM struct { coordinator *leader.Coordinator[asmClient] } -var _ Router[Secrets] = &ASM{} var _ Provider[Secrets] = &ASM{} func NewASM(ctx context.Context, secretsClient *secretsmanager.Client, advertise *url.URL, leaser leases.Leaser) *ASM { @@ -78,33 +74,6 @@ func (ASM) Key() string { return "asm" } -func (ASM) Get(ctx context.Context, ref Ref) (*url.URL, error) { - return asmURLForRef(ref), nil -} - -func (ASM) Set(ctx context.Context, ref Ref, key *url.URL) error { - expectedKey := asmURLForRef(ref) - if key.String() != expectedKey.String() { - return fmt.Errorf("key does not match expected key for ref: %s", expectedKey) - } - - return nil -} - -func (ASM) Unset(ctx context.Context, ref Ref) error { - // removing a secret is handled in Delete() - return nil -} - -// List all secrets in the account -func (a *ASM) List(ctx context.Context) ([]Entry, error) { - client, err := a.coordinator.Get() - if err != nil { - return nil, err - } - return client.list(ctx) -} - func (a *ASM) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { client, err := a.coordinator.Get() if err != nil { diff --git a/common/configuration/asm_test.go b/common/configuration/asm_test.go index 0c3db22835..8480f144c1 100644 --- a/common/configuration/asm_test.go +++ b/common/configuration/asm_test.go @@ -54,7 +54,8 @@ func TestASMWorkflow(t *testing.T) { asm, leader, _, _ := localstack(ctx, t) ref := Ref{Module: Some("foo"), Name: "bar"} var mySecret = jsonBytes(t, "my secret") - manager, err := New(ctx, asm, []Provider[Secrets]{asm}) + sr := NewDBSecretResolver(&mockDBSecretResolverDAL{}) + manager, err := New(ctx, sr, []Provider[Secrets]{asm}) assert.NoError(t, err) var gotSecret []byte @@ -102,7 +103,8 @@ func TestASMWorkflow(t *testing.T) { func TestASMPagination(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) asm, leader, _, _ := localstack(ctx, t) - manager, err := New(ctx, asm, []Provider[Secrets]{asm}) + sr := NewDBSecretResolver(&mockDBSecretResolverDAL{}) + manager, err := New(ctx, sr, []Provider[Secrets]{asm}) assert.NoError(t, err) // Create 210 secrets, so we paginate at least twice. @@ -341,7 +343,11 @@ func (c *fakeAdminClient) ConfigUnset(ctx context.Context, req *connect.Request[ } func (c *fakeAdminClient) SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) { - listing, err := c.asm.List(ctx) + client, err := c.asm.coordinator.Get() + if err != nil { + return nil, err + } + listing, err := client.list(ctx) if err != nil { return nil, err } diff --git a/common/configuration/dal/dal.go b/common/configuration/dal/dal.go index 0f592951d0..b8e106e6ba 100644 --- a/common/configuration/dal/dal.go +++ b/common/configuration/dal/dal.go @@ -3,6 +3,7 @@ package dal import ( "context" + "fmt" "github.com/alecthomas/types/optional" "github.com/jackc/pgx/v5/pgxpool" @@ -45,3 +46,44 @@ func (d *DAL) ListModuleConfiguration(ctx context.Context) ([]sql.ModuleConfigur } return l, nil } + +func (d *DAL) GetModuleSecretURL(ctx context.Context, module optional.Option[string], name string) (string, error) { + b, err := d.db.GetModuleSecretURL(ctx, module, name) + if err != nil { + return "", fmt.Errorf("could not get secret URL: %w", dalerrs.TranslatePGError(err)) + } + return b, nil +} + +func (d *DAL) SetModuleSecretURL(ctx context.Context, module optional.Option[string], name string, url string) error { + err := d.db.SetModuleSecretURL(ctx, module, name, url) + if err != nil { + return fmt.Errorf("could not set secret URL: %w", dalerrs.TranslatePGError(err)) + } + return nil +} + +func (d *DAL) UnsetModuleSecret(ctx context.Context, module optional.Option[string], name string) error { + err := d.db.UnsetModuleSecret(ctx, module, name) + if err != nil { + return fmt.Errorf("could not unset secret: %w", dalerrs.TranslatePGError(err)) + } + return nil +} + +type ModuleSecret sql.ModuleSecret + +func (d *DAL) ListModuleSecrets(ctx context.Context) ([]ModuleSecret, error) { + l, err := d.db.ListModuleSecrets(ctx) + if err != nil { + return nil, fmt.Errorf("could not list secrets: %w", dalerrs.TranslatePGError(err)) + } + + // Convert []sql.ModuleSecret to []ModuleSecret + ms := make([]ModuleSecret, len(l)) + for i, secret := range l { + ms[i] = ModuleSecret(secret) + } + + return ms, nil +} diff --git a/common/configuration/db_secret_resolver.go b/common/configuration/db_secret_resolver.go new file mode 100644 index 0000000000..6e863d0e47 --- /dev/null +++ b/common/configuration/db_secret_resolver.go @@ -0,0 +1,81 @@ +package configuration + +import ( + "context" + "fmt" + "net/url" + + "github.com/TBD54566975/ftl/common/configuration/dal" + "github.com/alecthomas/types/optional" +) + +// DBSecretResolver loads values a project's secrets from the given database. +type DBSecretResolver struct { + dal DBSecretResolverDAL +} + +type DBSecretResolverDAL interface { + GetModuleSecretURL(ctx context.Context, module optional.Option[string], name string) (string, error) + ListModuleSecrets(ctx context.Context) ([]dal.ModuleSecret, error) + SetModuleSecretURL(ctx context.Context, module optional.Option[string], name string, url string) error + UnsetModuleSecret(ctx context.Context, module optional.Option[string], name string) error +} + +// DBSecretResolver should only be used for secrets +var _ Router[Secrets] = DBSecretResolver{} + +func NewDBSecretResolver(db DBSecretResolverDAL) DBSecretResolver { + return DBSecretResolver{dal: db} +} + +func (d DBSecretResolver) Role() Secrets { return Secrets{} } + +func (d DBSecretResolver) Get(ctx context.Context, ref Ref) (*url.URL, error) { + u, err := d.dal.GetModuleSecretURL(ctx, ref.Module, ref.Name) + if err != nil { + return nil, fmt.Errorf("failed to get secret URL: %w", err) + } + url, err := url.Parse(u) + if err != nil { + return nil, fmt.Errorf("failed to parse secret URL: %w", err) + } + return url, nil +} + +func (d DBSecretResolver) List(ctx context.Context) ([]Entry, error) { + secrets, err := d.dal.ListModuleSecrets(ctx) + if err != nil { + return nil, fmt.Errorf("could not list module secrets: %w", err) + } + entries := make([]Entry, len(secrets)) + for i, s := range secrets { + url, err := url.Parse(s.Url) + if err != nil { + return nil, fmt.Errorf("failed to parse secret URL: %w", err) + } + entries[i] = Entry{ + Ref: Ref{ + Module: s.Module, + Name: s.Name, + }, + Accessor: url, + } + } + return entries, nil +} + +func (d DBSecretResolver) Set(ctx context.Context, ref Ref, key *url.URL) error { + err := d.dal.SetModuleSecretURL(ctx, ref.Module, ref.Name, key.String()) + if err != nil { + return fmt.Errorf("failed to set secret URL: %w", err) + } + return nil +} + +func (d DBSecretResolver) Unset(ctx context.Context, ref Ref) error { + err := d.dal.UnsetModuleSecret(ctx, ref.Module, ref.Name) + if err != nil { + return fmt.Errorf("failed to unset secret: %w", err) + } + return nil +} diff --git a/common/configuration/db_secret_resolver_test.go b/common/configuration/db_secret_resolver_test.go new file mode 100644 index 0000000000..93c652c740 --- /dev/null +++ b/common/configuration/db_secret_resolver_test.go @@ -0,0 +1,82 @@ +package configuration + +import ( + "context" + "fmt" + "net/url" + "testing" + + "github.com/TBD54566975/ftl/common/configuration/dal" + "github.com/alecthomas/assert/v2" + . "github.com/alecthomas/types/optional" +) + +type mockDBSecretResolverDAL struct { + entries []dal.ModuleSecret +} + +func (d *mockDBSecretResolverDAL) findEntry(module Option[string], name string) (Option[dal.ModuleSecret], int) { + for i := range d.entries { + if d.entries[i].Module.Default("") == module.Default("") && d.entries[i].Name == name { + return Some(d.entries[i]), i + } + } + return None[dal.ModuleSecret](), -1 +} + +func (d *mockDBSecretResolverDAL) GetModuleSecretURL(ctx context.Context, module Option[string], name string) (string, error) { + entry, _ := d.findEntry(module, name) + if e, ok := entry.Get(); ok { + return e.Url, nil + } + return "", fmt.Errorf("secret not found") +} + +func (d *mockDBSecretResolverDAL) ListModuleSecrets(ctx context.Context) ([]dal.ModuleSecret, error) { + return d.entries, nil +} + +func (d *mockDBSecretResolverDAL) SetModuleSecretURL(ctx context.Context, module Option[string], name string, url string) error { + err := d.UnsetModuleSecret(ctx, module, name) + if err != nil { + return fmt.Errorf("could not unset secret %w", err) + } + d.entries = append(d.entries, dal.ModuleSecret{Module: module, Name: name, Url: url}) + return nil +} + +func (d *mockDBSecretResolverDAL) UnsetModuleSecret(ctx context.Context, module Option[string], name string) error { + entry, i := d.findEntry(module, name) + if _, ok := entry.Get(); ok { + d.entries = append(d.entries[:i], d.entries[i+1:]...) + } + return nil +} + +func TestDBSecretResolverList(t *testing.T) { + ctx := context.Background() + resolver := NewDBSecretResolver(&mockDBSecretResolverDAL{}) + + rone := Ref{Module: Some("foo"), Name: "one"} + err := resolver.Set(ctx, rone, &url.URL{Scheme: "asm", Host: rone.String()}) + assert.NoError(t, err) + + rtwo := Ref{Module: Some("foo"), Name: "two"} + err = resolver.Set(ctx, rtwo, &url.URL{Scheme: "asm", Host: rtwo.String()}) + assert.NoError(t, err) + + entries, err := resolver.List(ctx) + assert.NoError(t, err) + assert.Equal(t, len(entries), 2) + + err = resolver.Unset(ctx, rone) + assert.NoError(t, err) + + entries, err = resolver.List(ctx) + assert.NoError(t, err) + assert.Equal(t, len(entries), 1) + + url, err := resolver.Get(ctx, rtwo) + assert.NoError(t, err) + assert.Equal(t, url.String(), "asm://foo.two") +} diff --git a/common/configuration/sql/models.go b/common/configuration/sql/models.go index 6d2095f7ba..8ab61783d8 100644 --- a/common/configuration/sql/models.go +++ b/common/configuration/sql/models.go @@ -478,6 +478,14 @@ type ModuleConfiguration struct { Value []byte } +type ModuleSecret struct { + ID int64 + CreatedAt time.Time + Module optional.Option[string] + Name string + Url string +} + type Request struct { ID int64 Origin Origin diff --git a/common/configuration/sql/querier.go b/common/configuration/sql/querier.go index 17b7d75e74..adfbe2f0b1 100644 --- a/common/configuration/sql/querier.go +++ b/common/configuration/sql/querier.go @@ -12,9 +12,13 @@ import ( type Querier interface { GetModuleConfiguration(ctx context.Context, module optional.Option[string], name string) ([]byte, error) + GetModuleSecretURL(ctx context.Context, module optional.Option[string], name string) (string, error) ListModuleConfiguration(ctx context.Context) ([]ModuleConfiguration, error) + ListModuleSecrets(ctx context.Context) ([]ModuleSecret, error) SetModuleConfiguration(ctx context.Context, module optional.Option[string], name string, value []byte) error + SetModuleSecretURL(ctx context.Context, module optional.Option[string], name string, url string) error UnsetModuleConfiguration(ctx context.Context, module optional.Option[string], name string) error + UnsetModuleSecret(ctx context.Context, module optional.Option[string], name string) error } var _ Querier = (*Queries)(nil) diff --git a/common/configuration/sql/queries.sql b/common/configuration/sql/queries.sql index bf8835e2f3..d90ffbd02e 100644 --- a/common/configuration/sql/queries.sql +++ b/common/configuration/sql/queries.sql @@ -19,3 +19,25 @@ VALUES ($1, $2, $3); -- name: UnsetModuleConfiguration :exec DELETE FROM module_configuration WHERE module = @module AND name = @name; + +-- name: GetModuleSecretURL :one +SELECT url +FROM module_secrets +WHERE + (module IS NULL OR module = @module) + AND name = @name +ORDER BY module NULLS LAST +LIMIT 1; + +-- name: ListModuleSecrets :many +SELECT * +FROM module_secrets +ORDER BY module, name; + +-- name: SetModuleSecretURL :exec +INSERT INTO module_secrets (module, name, url) +VALUES ($1, $2, $3); + +-- name: UnsetModuleSecret :exec +DELETE FROM module_secrets +WHERE module = @module AND name = @name; \ No newline at end of file diff --git a/common/configuration/sql/queries.sql.go b/common/configuration/sql/queries.sql.go index 9a640d3efa..35850ae4e5 100644 --- a/common/configuration/sql/queries.sql.go +++ b/common/configuration/sql/queries.sql.go @@ -28,6 +28,23 @@ func (q *Queries) GetModuleConfiguration(ctx context.Context, module optional.Op return value, err } +const getModuleSecretURL = `-- name: GetModuleSecretURL :one +SELECT url +FROM module_secrets +WHERE + (module IS NULL OR module = $1) + AND name = $2 +ORDER BY module NULLS LAST +LIMIT 1 +` + +func (q *Queries) GetModuleSecretURL(ctx context.Context, module optional.Option[string], name string) (string, error) { + row := q.db.QueryRow(ctx, getModuleSecretURL, module, name) + var url string + err := row.Scan(&url) + return url, err +} + const listModuleConfiguration = `-- name: ListModuleConfiguration :many SELECT id, created_at, module, name, value FROM module_configuration @@ -60,6 +77,38 @@ func (q *Queries) ListModuleConfiguration(ctx context.Context) ([]ModuleConfigur return items, nil } +const listModuleSecrets = `-- name: ListModuleSecrets :many +SELECT id, created_at, module, name, url +FROM module_secrets +ORDER BY module, name +` + +func (q *Queries) ListModuleSecrets(ctx context.Context) ([]ModuleSecret, error) { + rows, err := q.db.Query(ctx, listModuleSecrets) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ModuleSecret + for rows.Next() { + var i ModuleSecret + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.Module, + &i.Name, + &i.Url, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const setModuleConfiguration = `-- name: SetModuleConfiguration :exec INSERT INTO module_configuration (module, name, value) VALUES ($1, $2, $3) @@ -70,6 +119,16 @@ func (q *Queries) SetModuleConfiguration(ctx context.Context, module optional.Op return err } +const setModuleSecretURL = `-- name: SetModuleSecretURL :exec +INSERT INTO module_secrets (module, name, url) +VALUES ($1, $2, $3) +` + +func (q *Queries) SetModuleSecretURL(ctx context.Context, module optional.Option[string], name string, url string) error { + _, err := q.db.Exec(ctx, setModuleSecretURL, module, name, url) + return err +} + const unsetModuleConfiguration = `-- name: UnsetModuleConfiguration :exec DELETE FROM module_configuration WHERE module = $1 AND name = $2 @@ -79,3 +138,13 @@ func (q *Queries) UnsetModuleConfiguration(ctx context.Context, module optional. _, err := q.db.Exec(ctx, unsetModuleConfiguration, module, name) return err } + +const unsetModuleSecret = `-- name: UnsetModuleSecret :exec +DELETE FROM module_secrets +WHERE module = $1 AND name = $2 +` + +func (q *Queries) UnsetModuleSecret(ctx context.Context, module optional.Option[string], name string) error { + _, err := q.db.Exec(ctx, unsetModuleSecret, module, name) + return err +} From 3c1f61bc76f0ded9e4917b77e9180ba65b6520e8 Mon Sep 17 00:00:00 2001 From: gak Date: Thu, 4 Jul 2024 09:32:40 +1000 Subject: [PATCH 02/23] fix: vsc extenion release version needs "v" strip (#1961) --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0652c11c9..1700d92701 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -174,9 +174,9 @@ jobs: - name: Publish run: | set -euo pipefail - version="$(git describe --tags --abbrev=0)" + version="$(git describe --tags --abbrev=0 | sed 's/^v//')" echo "Publishing version $version" - jq ".version = \"$version\"" extensions/vscode/package.json > extensions/vscode/package.json.tmp + jq --arg version "$version" '.version = $version' extensions/vscode/package.json > extensions/vscode/package.json.tmp mv extensions/vscode/package.json.tmp extensions/vscode/package.json just publish-extension env: From c5ddd13606e312329e1ec95f6f937a0e2244254e Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 4 Jul 2024 10:00:49 +1000 Subject: [PATCH 03/23] fix: validate project config (#1955) Validates project config files. This breaks previous cases where `ftl config set` and similar would be able to create the project toml if it wasn't already created. Now that a project's name is required, an invalid project config would have been created. It is now a requirement to go through `ftl init ` --- backend/controller/admin/admin_test.go | 2 +- .../admin/testdata/ftl-project.toml | 1 + .../sql/testdata/go/database/ftl-project.toml | 1 + common/configuration/manager_test.go | 2 +- .../configuration/projectconfig_resolver.go | 4 +- .../projectconfig_resolver_test.go | 19 +++----- .../configuration/testdata/ftl-project.toml | 2 + common/projectconfig/projectconfig.go | 48 ++++++++++--------- common/projectconfig/projectconfig_test.go | 1 + .../projectconfig/testdata/ftl-project.toml | 1 + .../testdata/go/configs-ftl-project.toml | 2 + .../testdata/go/findconfig/ftl-project.toml | 2 + .../go/no-module-dirs-ftl-project.toml | 2 + .../testdata/withMinVersion/ftl-project.toml | 1 + .../go/wrapped/ftl-project-test-1.toml | 1 + .../testdata/go/wrapped/ftl-project.toml | 1 + integration/harness.go | 2 +- 17 files changed, 51 insertions(+), 41 deletions(-) diff --git a/backend/controller/admin/admin_test.go b/backend/controller/admin/admin_test.go index 036123a5a0..636d46044f 100644 --- a/backend/controller/admin/admin_test.go +++ b/backend/controller/admin/admin_test.go @@ -62,7 +62,7 @@ func tempConfigPath(t *testing.T, existingPath string, prefix string) string { var existing []byte var err error if existingPath == "" { - existing = []byte{} + existing = []byte(`name = "generated"`) } else { existing, err = os.ReadFile(existingPath) assert.NoError(t, err) diff --git a/backend/controller/admin/testdata/ftl-project.toml b/backend/controller/admin/testdata/ftl-project.toml index 0f2031dc05..c37ff76faf 100644 --- a/backend/controller/admin/testdata/ftl-project.toml +++ b/backend/controller/admin/testdata/ftl-project.toml @@ -1,3 +1,4 @@ +name = "testdata" [global] [global.secrets] bar = "envar://baza" diff --git a/backend/controller/sql/testdata/go/database/ftl-project.toml b/backend/controller/sql/testdata/go/database/ftl-project.toml index 5a03622d5f..29317cb900 100644 --- a/backend/controller/sql/testdata/go/database/ftl-project.toml +++ b/backend/controller/sql/testdata/go/database/ftl-project.toml @@ -1,3 +1,4 @@ +name = "testdata" ftl-min-version = "" [global] diff --git a/common/configuration/manager_test.go b/common/configuration/manager_test.go index 8626fd4fca..d7bf6b6830 100644 --- a/common/configuration/manager_test.go +++ b/common/configuration/manager_test.go @@ -101,7 +101,7 @@ func tempConfigPath(t *testing.T, existingPath string, prefix string) string { var existing []byte var err error if existingPath == "" { - existing = []byte{} + existing = []byte(`name = "generated"`) } else { existing, err = os.ReadFile(existingPath) assert.NoError(t, err) diff --git a/common/configuration/projectconfig_resolver.go b/common/configuration/projectconfig_resolver.go index d40521f680..0a84d72bc4 100644 --- a/common/configuration/projectconfig_resolver.go +++ b/common/configuration/projectconfig_resolver.go @@ -70,7 +70,7 @@ func (p ProjectConfigResolver[R]) List(ctx context.Context) ([]Entry, error) { } func (p ProjectConfigResolver[R]) Set(ctx context.Context, ref Ref, key *url.URL) error { - config, err := pc.LoadOrCreate(ctx, p.Config) + config, err := pc.Load(ctx, p.Config) if err != nil { return err } @@ -83,7 +83,7 @@ func (p ProjectConfigResolver[R]) Set(ctx context.Context, ref Ref, key *url.URL } func (p ProjectConfigResolver[From]) Unset(ctx context.Context, ref Ref) error { - config, err := pc.LoadOrCreate(ctx, p.Config) + config, err := pc.Load(ctx, p.Config) if err != nil { return err } diff --git a/common/configuration/projectconfig_resolver_test.go b/common/configuration/projectconfig_resolver_test.go index d4beb6d73a..ed9a6f1b6d 100644 --- a/common/configuration/projectconfig_resolver_test.go +++ b/common/configuration/projectconfig_resolver_test.go @@ -26,19 +26,12 @@ func TestSet(t *testing.T) { err = os.WriteFile(config, existing, 0600) assert.NoError(t, err) - t.Run("ExistingModule", func(t *testing.T) { - setAndAssert(t, "echo", config) - }) - t.Run("NewModule", func(t *testing.T) { - setAndAssert(t, "echooo", config) - }) - t.Run("MissingTOMLFile", func(t *testing.T) { - err := os.Remove(config) - assert.NoError(t, err) - setAndAssert(t, "echooooo", config) - err = os.WriteFile(defaultPath, origConfigBytes, 0600) - assert.NoError(t, err) - }) + setAndAssert(t, "echo", config) + setAndAssert(t, "echooo", config) + + // Restore the original config file. + err = os.WriteFile(defaultPath, origConfigBytes, 0600) + assert.NoError(t, err) } func TestGetGlobal(t *testing.T) { diff --git a/common/configuration/testdata/ftl-project.toml b/common/configuration/testdata/ftl-project.toml index 962394b673..eac958e49d 100644 --- a/common/configuration/testdata/ftl-project.toml +++ b/common/configuration/testdata/ftl-project.toml @@ -1,3 +1,5 @@ +name = "testdata" + [global] [global.secrets] baz = "envar://baz" diff --git a/common/projectconfig/projectconfig.go b/common/projectconfig/projectconfig.go index 316cdf56d7..cc5cfdbe63 100644 --- a/common/projectconfig/projectconfig.go +++ b/common/projectconfig/projectconfig.go @@ -47,6 +47,26 @@ func (c Config) Root() string { return filepath.Dir(c.Path) } +// Validate checks that the configuration is valid. +func (c Config) Validate() error { + if c.Name == "" { + return fmt.Errorf("project name is required: %s", c.Path) + } + if strings.Contains(c.Name, " ") { + return fmt.Errorf("project name %q includes spaces: %s", c.Name, c.Path) + } + if c.FTLMinVersion != "" && !ftl.IsVersionAtLeastMin(ftl.Version, c.FTLMinVersion) { + return fmt.Errorf("FTL version %q predates the minimum version %q", ftl.Version, c.FTLMinVersion) + } + for _, dir := range c.ModuleDirs { + absDir := filepath.Clean(filepath.Join(c.Root(), dir)) + if !strings.HasPrefix(absDir, c.Root()) { + return fmt.Errorf("module-dirs path %q is not within the project root %q", dir, c.Root()) + } + } + return nil +} + // AbsModuleDirs returns the absolute path for the module-dirs field from the ftl-project.toml, unless // that is not defined, in which case it defaults to the root directory. func (c Config) AbsModuleDirs() []string { @@ -103,6 +123,9 @@ func DefaultConfigPath() optional.Option[string] { // Create creates the ftl-project.toml file with the given Config into dir. func Create(ctx context.Context, config Config, dir string) error { + if err := config.Validate(); err != nil { + return err + } logger := log.FromContext(ctx) path, err := filepath.Abs(dir) if err != nil { @@ -121,19 +144,6 @@ func Create(ctx context.Context, config Config, dir string) error { return Save(config) } -// LoadOrCreate loads or creates the given configuration file. -func LoadOrCreate(ctx context.Context, target string) (Config, error) { - logger := log.FromContext(ctx) - if _, err := os.Stat(target); errors.Is(err, os.ErrNotExist) { - logger.Debugf("Creating a new project config file at %q", target) - config := Config{Path: target} - return config, Save(config) - } - - log.FromContext(ctx).Tracef("Loading config from %s", target) - return Load(ctx, target) -} - // Load project config from a file. func Load(ctx context.Context, path string) (Config, error) { if path == "" { @@ -159,19 +169,11 @@ func Load(ctx context.Context, path string) (Config, error) { } return Config{}, fmt.Errorf("unknown configuration keys: %s", strings.Join(keys, ", ")) } - - if config.FTLMinVersion != "" && !ftl.IsVersionAtLeastMin(ftl.Version, config.FTLMinVersion) { - return config, fmt.Errorf("FTL version %q predates the minimum version %q", ftl.Version, config.FTLMinVersion) - } config.Path = path - for _, dir := range config.ModuleDirs { - absDir := filepath.Clean(filepath.Join(config.Root(), dir)) - if !strings.HasPrefix(absDir, config.Root()) { - return Config{}, fmt.Errorf("module-dirs path %q is not within the project root %q", dir, config.Root()) - } + if err := config.Validate(); err != nil { + return Config{}, err } - return config, nil } diff --git a/common/projectconfig/projectconfig_test.go b/common/projectconfig/projectconfig_test.go index 40e0d9a124..db5b35f7d8 100644 --- a/common/projectconfig/projectconfig_test.go +++ b/common/projectconfig/projectconfig_test.go @@ -14,6 +14,7 @@ func TestProjectConfig(t *testing.T) { actual, err := Load(context.Background(), "testdata/ftl-project.toml") assert.NoError(t, err) expected := Config{ + Name: "testdata", Path: actual.Path, Modules: map[string]ConfigAndSecrets{ "module": { diff --git a/common/projectconfig/testdata/ftl-project.toml b/common/projectconfig/testdata/ftl-project.toml index 16bb59dbaf..ab80490035 100644 --- a/common/projectconfig/testdata/ftl-project.toml +++ b/common/projectconfig/testdata/ftl-project.toml @@ -1,3 +1,4 @@ +name = "testdata" module-dirs = ["a/b/c", "d"] [modules.module.configuration] diff --git a/common/projectconfig/testdata/go/configs-ftl-project.toml b/common/projectconfig/testdata/go/configs-ftl-project.toml index 06f80b7af5..4ce8cbd2cd 100644 --- a/common/projectconfig/testdata/go/configs-ftl-project.toml +++ b/common/projectconfig/testdata/go/configs-ftl-project.toml @@ -1,3 +1,5 @@ +name = "testdata" + [global] [global.configuration] key = "inline://InZhbHVlIg" diff --git a/common/projectconfig/testdata/go/findconfig/ftl-project.toml b/common/projectconfig/testdata/go/findconfig/ftl-project.toml index 08c214312d..5de993f307 100644 --- a/common/projectconfig/testdata/go/findconfig/ftl-project.toml +++ b/common/projectconfig/testdata/go/findconfig/ftl-project.toml @@ -1,3 +1,5 @@ +name = "testdata" + [global] [global.configuration] test = "inline://InRlc3Qi" diff --git a/common/projectconfig/testdata/go/no-module-dirs-ftl-project.toml b/common/projectconfig/testdata/go/no-module-dirs-ftl-project.toml index e5b05afd26..09989a6796 100644 --- a/common/projectconfig/testdata/go/no-module-dirs-ftl-project.toml +++ b/common/projectconfig/testdata/go/no-module-dirs-ftl-project.toml @@ -1,2 +1,4 @@ +name = "testdata" + [commands] startup = ["echo 'Executing global pre-build command'"] diff --git a/common/projectconfig/testdata/withMinVersion/ftl-project.toml b/common/projectconfig/testdata/withMinVersion/ftl-project.toml index fc1aa653af..86e615be27 100644 --- a/common/projectconfig/testdata/withMinVersion/ftl-project.toml +++ b/common/projectconfig/testdata/withMinVersion/ftl-project.toml @@ -1,3 +1,4 @@ +name = "testdata" module-dirs = ["a/b/c", "d"] ftl-min-version = "0.129.2" diff --git a/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project-test-1.toml b/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project-test-1.toml index 66b669da9c..564a3f1ff9 100644 --- a/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project-test-1.toml +++ b/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project-test-1.toml @@ -1,3 +1,4 @@ +name = "testdata" ftl-min-version = "" [global] diff --git a/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project.toml b/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project.toml index 047314025e..38963fc060 100644 --- a/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project.toml +++ b/go-runtime/ftl/ftltest/testdata/go/wrapped/ftl-project.toml @@ -1,3 +1,4 @@ +name = "testdata" ftl-min-version = "" [global] diff --git a/integration/harness.go b/integration/harness.go index a293e101f4..d4c55fc955 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -79,7 +79,7 @@ func run(t *testing.T, ftlConfigPath string, startController bool, actions ...Ac // is used by FTL during startup. t.Setenv("FTL_CONFIG", filepath.Join(cwd, "testdata", "go", ftlConfigPath)) } else { - err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte{}, 0644) + err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644) assert.NoError(t, err) } From eb1410602104c5374983de5dad507258f3985a3b Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 4 Jul 2024 10:23:30 +1000 Subject: [PATCH 04/23] test: disable TestBox due to timeouts (#1965) --- cmd/ftl/integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/ftl/integration_test.go b/cmd/ftl/integration_test.go index c2fc0e5547..68354920dd 100644 --- a/cmd/ftl/integration_test.go +++ b/cmd/ftl/integration_test.go @@ -14,6 +14,8 @@ import ( ) func TestBox(t *testing.T) { + t.Skip("skipping due to timeouts") + // Need a longer timeout to wait for FTL inside Docker. t.Setenv("FTL_INTEGRATION_TEST_TIMEOUT", "30s") Infof("Building local ftl0/ftl-box:latest Docker image") From 5b8e2852e7d755add381e4f6ff06c123ea6e0bba Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 4 Jul 2024 10:32:16 +1000 Subject: [PATCH 05/23] test: TestSingleLeader data race fixed (#1964) fixes: #1963 --- backend/controller/leader/leader_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/controller/leader/leader_test.go b/backend/controller/leader/leader_test.go index 683fe174a7..7d239b8a41 100644 --- a/backend/controller/leader/leader_test.go +++ b/backend/controller/leader/leader_test.go @@ -102,8 +102,11 @@ func TestSingleLeader(t *testing.T) { assert.NoError(t, err) // replace original leading coordinator's advertising url to ensure the leader url changes + coordinators[leaderIdx].mutex.Lock() coordinators[leaderIdx].advertise, err = url.Parse("http://localhost:9999") assert.NoError(t, err) + coordinators[leaderIdx].mutex.Unlock() + time.Sleep(leaseTTL + time.Millisecond*500) leaderIdx, finalLeaderStr := leaderFromCoordinators(t, coordinators) From 8bb668262c09bf8742b70d4683f0122cde6ddc5c Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 4 Jul 2024 10:39:08 +1000 Subject: [PATCH 06/23] test: fix toml in integration test (#1966) --- backend/controller/admin/testdata/go/ftl-project-dr.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/controller/admin/testdata/go/ftl-project-dr.toml b/backend/controller/admin/testdata/go/ftl-project-dr.toml index 1997b49be9..a65b02f7ff 100644 --- a/backend/controller/admin/testdata/go/ftl-project-dr.toml +++ b/backend/controller/admin/testdata/go/ftl-project-dr.toml @@ -1,3 +1,4 @@ +name = "dr" ftl-min-version = "dev" hermit = false no-git = false From 55a4e96acd8d66203c4cb6e0a3d30f0b61c9f8e3 Mon Sep 17 00:00:00 2001 From: gak Date: Thu, 4 Jul 2024 11:47:10 +1000 Subject: [PATCH 07/23] fix(vscode): url for repo update (#1967) Existing url on the extension page is old: image --- extensions/vscode/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index 98c7c488bd..8667a2aa42 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -7,7 +7,7 @@ "version": "0.0.0", "repository": { "type": "git", - "url": "https://github.com/TBD54566975/ftl-vscode" + "url": "https://github.com/TBD54566975/ftl" }, "engines": { "vscode": "^1.90.2" From bc7868a3ef019fb5502ad517793ebd28ee3e6f6e Mon Sep 17 00:00:00 2001 From: gak Date: Thu, 4 Jul 2024 13:45:09 +1000 Subject: [PATCH 08/23] ci: ensure frozen migrations (#1968) Fixes #1962 - Checks for changes to existing migration files against `main, ignoring new lines and single line comments. - Adds a `just ensure-frozen-migrations`. - Adds a CI check. --- .github/workflows/ci.yml | 12 +++++++ Justfile | 4 +++ lefthook.yml | 2 ++ scripts/ensure-frozen-migrations | 62 ++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100755 scripts/ensure-frozen-migrations diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2ffd1a6db..adb990ea82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,18 @@ jobs: run: just init-db - name: Vet SQL run: sqlc vet + ensure-frozen-migrations: + name: Ensure Frozen Migrations + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Init Hermit + uses: cashapp/activate-hermit@v1 + - name: Freeze Migrations + run: just ensure-frozen-migrations lint: name: Lint runs-on: ubuntu-latest diff --git a/Justfile b/Justfile index 2a128e369f..0f6d1eb3f3 100644 --- a/Justfile +++ b/Justfile @@ -133,6 +133,10 @@ test-readme *args: tidy: find . -name go.mod -execdir go mod tidy \; +# Check for changes in existing SQL migrations compared to main +ensure-frozen-migrations: + scripts/ensure-frozen-migrations + # Run backend tests test-backend: @gotestsum --hide-summary skipped --format-hide-empty-pkg -- -short -fullpath ./... diff --git a/lefthook.yml b/lefthook.yml index 1348bd6a5b..d8ecda5087 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -14,3 +14,5 @@ pre-push: lint-frontend: root: frontend/ run: just lint-frontend + ensure-frozen-migrations: + run: just ensure-frozen-migrations \ No newline at end of file diff --git a/scripts/ensure-frozen-migrations b/scripts/ensure-frozen-migrations new file mode 100755 index 0000000000..6c921f1e88 --- /dev/null +++ b/scripts/ensure-frozen-migrations @@ -0,0 +1,62 @@ +#!/bin/bash +set -euo pipefail + +# echo +set -x + +# This script checks if the SQL migration have been modified compared to the common ancestor with the main branch. +# It ignores comments when comparing the files. + +remove_comments() { + sed 's/[[:space:]]*--.*$//g' "$1" +} + +compare_sql_files() { + local file1="$1" + local file2="$2" + diff -q <(remove_comments "$file1" | sed '/^\s*$/d' | sort) \ + <(remove_comments "$file2" | sed '/^\s*$/d' | sort) > /dev/null +} + +show_diff() { + local file1="$1" + local file2="$2" + diff -u <(remove_comments "$file1") <(remove_comments "$file2") +} + +main() { + local sql_dir="backend/controller/sql/schema" + local changed_files + local merge_base + + # Find the merge base commit + merge_base=$(git merge-base HEAD origin/main) + + # Get list of changed SQL files compared to the merge base + changed_files=$(git diff --name-only "$merge_base" -- "$sql_dir"/*.sql) + + if [ -z "$changed_files" ]; then + echo "👍 No SQL files changed in $sql_dir compared to the common ancestor with main branch" + exit 0 + fi + + while IFS= read -r file; do + if [ ! -f "$file" ]; then + echo "$file no longer exists!" + exit 1 + fi + if compare_sql_files "$file" <(git show "$merge_base:$file"); then + : # Do nothing if files are the same + else + echo "❌ Migration files changes detected" + show_diff "$file" <(git show "$merge_base:$file") + exit 1 + fi + done <<< "$changed_files" + + git diff --color=always "$merge_base" -- "$sql_dir"/*.sql | cat + echo "👍 Only comments/new lines changed in SQL files" +} + +# Run the main function +main \ No newline at end of file From 0e17f9fff89c81e42d9fe7b2a6722257664df456 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Thu, 4 Jul 2024 14:38:10 +1000 Subject: [PATCH 09/23] feat: obfuscate secrets to avoid tampering (#1916) closes #1889 closes #1772 - Secrets manager has an obfuscator, config manager does not (`Secrets` conforms to `ObfuscatorProvider`) - `configuration.Obfuscator` allows obfuscating/revealing of values - These changes apply to all secrets providers - Providers can also save a comment warning about tampering - ASM does this by prefixing the secret with comments (lines starting with `#`) - 1Password will support comments in another PR --- common/configuration/asm_leader.go | 7 +-- common/configuration/comments.go | 32 +++++++++++ common/configuration/comments_test.go | 34 ++++++++++++ common/configuration/manager.go | 33 ++++++++++-- common/configuration/obfuscator.go | 69 ++++++++++++++++++++++++ common/configuration/obfuscator_test.go | 72 +++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 common/configuration/comments.go create mode 100644 common/configuration/comments_test.go create mode 100644 common/configuration/obfuscator.go create mode 100644 common/configuration/obfuscator_test.go diff --git a/common/configuration/asm_leader.go b/common/configuration/asm_leader.go index 141b47a8b3..286560ce01 100644 --- a/common/configuration/asm_leader.go +++ b/common/configuration/asm_leader.go @@ -123,7 +123,7 @@ func (l *asmLeader) sync(ctx context.Context, secrets *xsync.MapOf[Ref, cachedSe if s.SecretBinary != nil { return fmt.Errorf("secret for %s in ASM is not a string", ref) } - data := []byte(*s.SecretString) + data := unwrapComments([]byte(*s.SecretString)) secrets.Store(ref, cachedSecret{ value: data, versionToken: optional.Some[any](refsToLoad[ref]), @@ -155,9 +155,10 @@ func (l *asmLeader) load(ctx context.Context, ref Ref, key *url.URL) ([]byte, er // store and if the secret already exists, update it. func (l *asmLeader) store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + valueWithComments := aws.String(string(wrapWithComments(value, defaultSecretModificationWarning))) _, err := l.client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ Name: aws.String(ref.String()), - SecretString: aws.String(string(value)), + SecretString: valueWithComments, Tags: []types.Tag{ {Key: aws.String(asmTagKey), Value: aws.String(ref.Module.Default(""))}, }, @@ -168,7 +169,7 @@ func (l *asmLeader) store(ctx context.Context, ref Ref, value []byte) (*url.URL, if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceExistsException" { _, err = l.client.UpdateSecret(ctx, &secretsmanager.UpdateSecretInput{ SecretId: aws.String(ref.String()), - SecretString: aws.String(string(value)), + SecretString: valueWithComments, }) if err != nil { return nil, fmt.Errorf("unable to update secret in ASM: %w", err) diff --git a/common/configuration/comments.go b/common/configuration/comments.go new file mode 100644 index 0000000000..0fe1638658 --- /dev/null +++ b/common/configuration/comments.go @@ -0,0 +1,32 @@ +package configuration + +import ( + "strings" + + "github.com/TBD54566975/ftl/internal/slices" +) + +const defaultSecretModificationWarning = `This secret is managed by "ftl secret set", DO NOT MODIFY` + +// wrapWithComments wraps the secret with a comment to indicate that it is managed by FTL. +// +// This is used by providers that want to include a warning to avoid manual modification. +// The provider must support multiline secrets. +// Comment lines are prefixed with '# ' in the result. +func wrapWithComments(secret []byte, comments string) []byte { + lines := []string{} + for _, line := range strings.Split(comments, "\n") { + lines = append(lines, "# "+line) + } + lines = append(lines, string(secret)) + return []byte(strings.Join(lines, "\n")) +} + +// unwrapComments removes comments if they exist by looking for the lines starting with '#' +func unwrapComments(secret []byte) []byte { + lines := strings.Split(string(secret), "\n") + lines = slices.Filter(lines, func(line string) bool { + return !strings.HasPrefix(line, "#") + }) + return []byte(strings.Join(lines, "\n")) +} diff --git a/common/configuration/comments_test.go b/common/configuration/comments_test.go new file mode 100644 index 0000000000..384780e503 --- /dev/null +++ b/common/configuration/comments_test.go @@ -0,0 +1,34 @@ +package configuration + +import ( + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestComments(t *testing.T) { + for _, tt := range []struct { + input string + comment string + output string + }{ + { + input: "test input can be anything", + comment: "This is a test", + output: "# This is a test\ntest input can be anything", + }, + { + input: "{\n \"key\": \"value\"\n}", + comment: "This is a multi\nline\ncomment", + output: "# This is a multi\n# line\n# comment\n{\n \"key\": \"value\"\n}", + }, + } { + t.Run(tt.input, func(t *testing.T) { + wrapped := wrapWithComments([]byte(tt.input), tt.comment) + assert.Equal(t, tt.output, string(wrapped)) + + unwrapped := unwrapComments(wrapped) + assert.Equal(t, tt.input, string(unwrapped)) + }) + } +} diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 60ca5414c7..7490581f73 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -21,6 +21,12 @@ type Secrets struct{} func (Secrets) String() string { return "secrets" } +func (Secrets) obfuscator() Obfuscator { + return Obfuscator{ + key: []byte("obfuscatesecrets"), // 16 characters (AES-128), not meant to provide security + } +} + type Configuration struct{} func (Configuration) String() string { return "configuration" } @@ -28,8 +34,9 @@ func (Configuration) String() string { return "configuration" } // Manager is a high-level configuration manager that abstracts the details of // the Router and Provider interfaces. type Manager[R Role] struct { - providers map[string]Provider[R] - router Router[R] + providers map[string]Provider[R] + router Router[R] + obfuscator optional.Option[Obfuscator] } func ConfigFromEnvironment() []string { @@ -61,6 +68,9 @@ func New[R Role](ctx context.Context, router Router[R], providers []Provider[R]) for _, p := range providers { m.providers[p.Key()] = p } + if provider, ok := any(new(R)).(ObfuscatorProvider); ok { + m.obfuscator = optional.Some(provider.obfuscator()) + } m.router = router return m, nil } @@ -87,6 +97,12 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { if err != nil { return nil, fmt.Errorf("%s: %w", ref, err) } + if obfuscator, ok := m.obfuscator.Get(); ok { + data, err = obfuscator.Reveal(data) + if err != nil { + return nil, fmt.Errorf("could not reveal obfuscated value: %w", err) + } + } return data, nil } @@ -127,12 +143,23 @@ func (m *Manager[R]) SetJSON(ctx context.Context, pkey string, ref Ref, value js if err := checkJSON(value); err != nil { return fmt.Errorf("invalid value for %s, must be JSON: %w", m.router.Role(), err) } + var bytes []byte + if obfuscator, ok := m.obfuscator.Get(); ok { + var err error + bytes, err = obfuscator.Obfuscate(value) + if err != nil { + return fmt.Errorf("could not obfuscate: %w", err) + } + } else { + bytes = value + } + provider, ok := m.providers[pkey] if !ok { pkeys := strings.Join(m.availableProviderKeys(), ", ") return fmt.Errorf("no provider for key %q, specify one of: %s", pkey, pkeys) } - key, err := provider.Store(ctx, ref, value) + key, err := provider.Store(ctx, ref, bytes) if err != nil { return err } diff --git a/common/configuration/obfuscator.go b/common/configuration/obfuscator.go new file mode 100644 index 0000000000..8d940131f5 --- /dev/null +++ b/common/configuration/obfuscator.go @@ -0,0 +1,69 @@ +package configuration + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "io" + "strings" +) + +type ObfuscatorProvider interface { + obfuscator() Obfuscator +} + +// Obfuscator hides and reveals a value, but does not provide real security +// instead the aim of this Obfuscator is to make values not easily human readable +// +// Obfuscation is done by XOR-ing the input with the AES key. Length of key must be 16, 24 or 32 bytes (corresponding to AES-128, AES-192 or AES-256 keys). +type Obfuscator struct { + key []byte +} + +// Obfuscate takes a value and returns an obfuscated value (encoded in base64) +func (o Obfuscator) Obfuscate(input []byte) ([]byte, error) { + block, err := aes.NewCipher(o.key) + if err != nil { + return nil, fmt.Errorf("could not create cypher for obfuscation: %w", err) + } + ciphertext := make([]byte, aes.BlockSize+len(input)) + iv := ciphertext[:aes.BlockSize] + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return nil, fmt.Errorf("could not generate IV for obfuscation: %w", err) + } + cfb := cipher.NewCFBEncrypter(block, iv) + cfb.XORKeyStream(ciphertext[aes.BlockSize:], input) + return []byte(base64.StdEncoding.EncodeToString(ciphertext)), nil +} + +// Reveal takes an obfuscated value and de-obfuscates the base64 encoded value +func (o Obfuscator) Reveal(input []byte) ([]byte, error) { + // check if the input looks like it was obfuscated + if !strings.ContainsRune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/=", rune(input[0])) { + // known issue: an unobfuscated value which is just a number will be considered obfuscated + return input, nil + } + + obfuscated, err := base64.StdEncoding.DecodeString(string(input)) + if err != nil { + return nil, fmt.Errorf("expected hexadecimal string: %w", err) + } + block, err := aes.NewCipher(o.key) + if err != nil { + return nil, fmt.Errorf("could not create cypher for decoding obfuscation: %w", err) + } + if len(obfuscated) < aes.BlockSize { + return nil, errors.New("obfuscated value too short to decode") + } + iv := obfuscated[:aes.BlockSize] + obfuscated = obfuscated[aes.BlockSize:] + cfb := cipher.NewCFBDecrypter(block, iv) + + var output = make([]byte, len(obfuscated)) + cfb.XORKeyStream(output, obfuscated) + + return output, nil +} diff --git a/common/configuration/obfuscator_test.go b/common/configuration/obfuscator_test.go new file mode 100644 index 0000000000..7d33584a02 --- /dev/null +++ b/common/configuration/obfuscator_test.go @@ -0,0 +1,72 @@ +package configuration + +import ( + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/alecthomas/types/optional" +) + +func TestObfuscator(t *testing.T) { + defaultKey := []byte("1234567890123456") // 32 characters + for _, tt := range []struct { + input string + key []byte + expectedError optional.Option[string] + backwardsCompatible bool + }{ + { + input: "test input can be anything", + key: defaultKey, + backwardsCompatible: false, + }, + { + input: `"test input can be anything"`, + key: defaultKey, + backwardsCompatible: true, + }, + { + input: `"{\n "key": "value"\n}`, + key: defaultKey, + backwardsCompatible: true, + }, + { + input: `1.2345`, + key: defaultKey, + backwardsCompatible: false, + }, + { + input: "key is too short", + key: []byte("too-short"), + expectedError: optional.Some("could not create cypher for obfuscation: crypto/aes: invalid key size 9"), + }, + } { + t.Run(tt.input, func(t *testing.T) { + o := Obfuscator{ + key: tt.key, + } + // obfuscate + obfuscated, err := o.Obfuscate([]byte(tt.input)) + if expectedError, ok := tt.expectedError.Get(); ok { + assert.EqualError(t, err, expectedError) + return + } + assert.NoError(t, err) + + // reveal obfuscated value + revealed, err := o.Reveal(obfuscated) + assert.NoError(t, err) + assert.Equal(t, tt.input, string(revealed)) + + // obfuscated value should not include the input we are trying to obfuscate + assert.NotContains(t, string(obfuscated), tt.input) + + // reveal unobfuscated value to check backwards compatibility + if tt.backwardsCompatible { + revealed, err = o.Reveal([]byte(tt.input)) + assert.NoError(t, err) + assert.Equal(t, tt.input, string(revealed)) + } + }) + } +} From 7d1b490caf45493bbd7131a266d85272d4c1ae25 Mon Sep 17 00:00:00 2001 From: Jon Johnson <113393155+jonathanj-square@users.noreply.github.com> Date: Thu, 4 Jul 2024 02:51:36 -0700 Subject: [PATCH 10/23] fix: controller startup failure visibility (#1969) fixes #1959 controller startup failure troubleshooting was complicated by a failure to surface error messages from the `GetModuleContext` streaming end-point. errors encountered from that end-point are now made visible and if those errors are deemed fatal the runner will terminate. --- buildengine/engine.go | 2 +- internal/modulecontext/module_context.go | 51 +++++++++++++----- internal/modulecontext/module_context_test.go | 2 +- internal/rpc/rpc.go | 54 ++++++++++++------- .../modulecontext/module_context_utils.go | 2 +- 5 files changed, 76 insertions(+), 35 deletions(-) diff --git a/buildengine/engine.go b/buildengine/engine.go index 75fbc4e691..03894be43d 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -132,7 +132,7 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, modul return e, nil } schemaSync := e.startSchemaSync(ctx) - go rpc.RetryStreamingServerStream(ctx, backoff.Backoff{Max: time.Second}, &ftlv1.PullSchemaRequest{}, client.PullSchema, schemaSync) + go rpc.RetryStreamingServerStream(ctx, backoff.Backoff{Max: time.Second}, &ftlv1.PullSchemaRequest{}, client.PullSchema, schemaSync, rpc.AlwaysRetry()) return e, nil } diff --git a/internal/modulecontext/module_context.go b/internal/modulecontext/module_context.go index e3783c9177..f870de1a0b 100644 --- a/internal/modulecontext/module_context.go +++ b/internal/modulecontext/module_context.go @@ -1,9 +1,11 @@ package modulecontext import ( + "connectrpc.com/connect" "context" "database/sql" "encoding/json" + "errors" "fmt" "strings" "sync" @@ -180,7 +182,7 @@ func (m ModuleContext) BehaviorForVerb(ref schema.Ref) (optional.Option[VerbBeha } type ModuleContextSupplier interface { - Subscribe(ctx context.Context, moduleName string, sink func(ctx context.Context, moduleContext ModuleContext)) + Subscribe(ctx context.Context, moduleName string, sink func(ctx context.Context, moduleContext ModuleContext), errorRetryCallback func(err error) bool) } type grpcModuleContextSupplier struct { @@ -191,7 +193,7 @@ func NewModuleContextSupplier(client ftlv1connect.VerbServiceClient) ModuleConte return ModuleContextSupplier(grpcModuleContextSupplier{client}) } -func (g grpcModuleContextSupplier) Subscribe(ctx context.Context, moduleName string, sink func(ctx context.Context, moduleContext ModuleContext)) { +func (g grpcModuleContextSupplier) Subscribe(ctx context.Context, moduleName string, sink func(ctx context.Context, moduleContext ModuleContext), errorRetryCallback func(err error) bool) { request := &ftlv1.ModuleContextRequest{Module: moduleName} callback := func(_ context.Context, resp *ftlv1.ModuleContextResponse) error { mc, err := FromProto(resp) @@ -201,7 +203,7 @@ func (g grpcModuleContextSupplier) Subscribe(ctx context.Context, moduleName str sink(ctx, mc) return nil } - go rpc.RetryStreamingServerStream(ctx, backoff.Backoff{}, request, g.client.GetModuleContext, callback) + go rpc.RetryStreamingServerStream(ctx, backoff.Backoff{}, request, g.client.GetModuleContext, callback, errorRetryCallback) } // NewDynamicContext creates a new DynamicModuleContext. This operation blocks @@ -217,22 +219,43 @@ func NewDynamicContext(ctx context.Context, supplier ModuleContextSupplier, modu await.Add(1) releaseOnce := sync.Once{} + ctx, cancel := context.WithCancelCause(ctx) + deadline, timeoutCancel := context.WithTimeout(ctx, 5*time.Second) + g, _ := errgroup.WithContext(deadline) + defer timeoutCancel() + // asynchronously consumes a subscription of ModuleContext changes and signals the arrival of the first - supplier.Subscribe(ctx, moduleName, func(ctx context.Context, moduleContext ModuleContext) { - result.current.Store(moduleContext) - releaseOnce.Do(func() { - await.Done() + supplier.Subscribe( + ctx, + moduleName, + func(ctx context.Context, moduleContext ModuleContext) { + result.current.Store(moduleContext) + releaseOnce.Do(func() { + await.Done() + }) + }, + func(err error) bool { + var connectErr *connect.Error + + if errors.As(err, &connectErr) && connectErr.Code() == connect.CodeInternal { + cancel(err) + await.Done() + return false + } + + return true }) - }) - deadline, cancellation := context.WithTimeout(ctx, 5*time.Second) - g, _ := errgroup.WithContext(deadline) - defer cancellation() - - // wait for first ModuleContext to be made available (with the above timeout) + // await the WaitGroup's completion which either signals the availability of the + // first ModuleContext or an error g.Go(func() error { await.Wait() - return nil + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } }) if err := g.Wait(); err != nil { diff --git a/internal/modulecontext/module_context_test.go b/internal/modulecontext/module_context_test.go index acf6900973..eda23dd126 100644 --- a/internal/modulecontext/module_context_test.go +++ b/internal/modulecontext/module_context_test.go @@ -36,7 +36,7 @@ func TestDynamicContextUpdate(t *testing.T) { assert.Equal(t, mc2, dynamic.CurrentContext()) } -func (mcs *manualContextSupplier) Subscribe(ctx context.Context, _ string, sink func(ctx context.Context, mCtx ModuleContext)) { +func (mcs *manualContextSupplier) Subscribe(ctx context.Context, _ string, sink func(ctx context.Context, mCtx ModuleContext), _ func(error) bool) { sink(ctx, mcs.initialCtx) mcs.sink = sink } diff --git a/internal/rpc/rpc.go b/internal/rpc/rpc.go index ac21bdeb98..33f8a9bbda 100644 --- a/internal/rpc/rpc.go +++ b/internal/rpc/rpc.go @@ -195,6 +195,12 @@ func RetryStreamingClientStream[Req, Resp any]( } } +// AlwaysRetry instructs RetryStreamingServerStream to always retry the errors it encounters when +// supplied as the errorRetryCallback argument +func AlwaysRetry() func(error) bool { + return func(err error) bool { return true } +} + // RetryStreamingServerStream will repeatedly call handler with responses from // the stream returned by "rpc" until handler returns an error or the context is // cancelled. @@ -204,6 +210,7 @@ func RetryStreamingServerStream[Req, Resp any]( req *Req, rpc func(context.Context, *connect.Request[Req]) (*connect.ServerStreamForClient[Resp], error), handler func(ctx context.Context, resp *Resp) error, + errorRetryCallback func(err error) bool, ) { logLevel := log.Debug errored := false @@ -211,36 +218,47 @@ func RetryStreamingServerStream[Req, Resp any]( for { stream, err := rpc(ctx, connect.NewRequest(req)) if err == nil { - for stream.Receive() { - req := stream.Msg() - err = handler(ctx, req) - if err != nil { + for { + if stream.Receive() { + resp := stream.Msg() + err = handler(ctx, resp) + + if err != nil { + break + } + if errored { + logger.Debugf("Server stream recovered") + errored = false + } + select { + case <-ctx.Done(): + return + default: + } + retry.Reset() + logLevel = log.Warn + } else { + // Stream terminated; check if this was caused by an error + err = stream.Err() + logLevel = log.Warn break } - if errored { - logger.Debugf("Server stream recovered") - errored = false - } - select { - case <-ctx.Done(): - return - default: - } - retry.Reset() - logLevel = log.Warn } } - if err != nil { - err = stream.Err() - } errored = true delay := retry.Duration() if err != nil && !errors.Is(err, context.Canceled) { + if errorRetryCallback != nil && !errorRetryCallback(err) { + logger.Errorf(err, "Stream handler encountered a non-retryable error") + return + } + logger.Logf(logLevel, "Stream handler failed, retrying in %s: %s", delay, err) } else if err == nil { logger.Debugf("Stream finished, retrying in %s", delay) } + select { case <-ctx.Done(): return diff --git a/testutils/modulecontext/module_context_utils.go b/testutils/modulecontext/module_context_utils.go index bfd7d6a734..acf1a61d9f 100644 --- a/testutils/modulecontext/module_context_utils.go +++ b/testutils/modulecontext/module_context_utils.go @@ -22,6 +22,6 @@ func MakeDynamic(ctx context.Context, m modulecontext.ModuleContext) *modulecont return result } -func (smc SingleContextSupplier) Subscribe(ctx context.Context, _ string, sink func(ctx context.Context, mCtx modulecontext.ModuleContext)) { +func (smc SingleContextSupplier) Subscribe(ctx context.Context, _ string, sink func(ctx context.Context, mCtx modulecontext.ModuleContext), _ func(error) bool) { sink(ctx, smc.moduleCtx) } From 0a69db1b5db11d84538892ec525bddd00be0a5ef Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 4 Jul 2024 03:03:23 -0700 Subject: [PATCH 11/23] fix: use shared .ftl folder for stubs (#1843) Fixes #1662 Fixes #1827 --- .gitignore | 6 + Justfile | 2 +- buildengine/build.go | 8 +- buildengine/build_go.go | 4 +- buildengine/build_go_test.go | 249 +---------------- buildengine/build_test.go | 36 ++- buildengine/deploy_test.go | 11 +- buildengine/discover_test.go | 14 +- buildengine/engine.go | 45 +++- buildengine/engine_test.go | 4 +- buildengine/stubs.go | 26 ++ buildengine/stubs_test.go | 250 ++++++++++++++++++ cmd/ftl/cmd_box.go | 2 +- cmd/ftl/cmd_box_run.go | 5 +- cmd/ftl/cmd_build.go | 2 +- cmd/ftl/cmd_deploy.go | 5 +- cmd/ftl/cmd_dev.go | 2 +- cmd/ftl/cmd_init.go | 4 +- cmd/ftl/cmd_new.go | 10 +- common/moduleconfig/moduleconfig.go | 2 +- extensions/vscode/package-lock.json | 4 +- .../go/main/go.mod.tmpl | 2 +- .../go/main/main.go.tmpl} | 0 .../compile/build-template/go.work.tmpl | 6 +- go-runtime/compile/build.go | 182 ++++++++----- go-runtime/compile/devel.go | 5 + .../external_module.go.tmpl | 32 +-- .../modules/{{ .Module.Name }}}/go.mod.tmpl | 4 +- .../compile/external-module-template/go.mod | 3 - .../external-module-template/go.work.tmpl | 6 - .../compile/main-work-template/go.work.tmpl | 8 + go-runtime/compile/release.go | 11 + go-runtime/compile/schema_test.go | 8 +- .../{{ .Name | camel | lower }}/go.mod.tmpl | 4 +- integration/harness.go | 23 +- 35 files changed, 595 insertions(+), 390 deletions(-) create mode 100644 buildengine/stubs.go create mode 100644 buildengine/stubs_test.go rename go-runtime/compile/build-template/{_ftl.tmpl => .ftl.tmpl}/go/main/go.mod.tmpl (94%) rename go-runtime/compile/build-template/{_ftl.tmpl/go/main/main.go => .ftl.tmpl/go/main/main.go.tmpl} (100%) rename go-runtime/compile/external-module-template/{_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }} => .ftl/go/modules/{{ .Module.Name }}}/external_module.go.tmpl (66%) rename go-runtime/compile/external-module-template/{_ftl/go/modules => .ftl/go/modules/{{ .Module.Name }}}/go.mod.tmpl (80%) delete mode 100644 go-runtime/compile/external-module-template/go.mod delete mode 100644 go-runtime/compile/external-module-template/go.work.tmpl create mode 100644 go-runtime/compile/main-work-template/go.work.tmpl diff --git a/.gitignore b/.gitignore index e54ce37176..31dcf44481 100644 --- a/.gitignore +++ b/.gitignore @@ -20,9 +20,15 @@ examples/**/go.work examples/**/go.work.sum testdata/**/go.work testdata/**/go.work.sum + +# Leaving old _ftl for now to avoid old stuff getting checked in **/testdata/**/_ftl **/examples/**/_ftl **/examples/**/types.ftl.go +**/testdata/**/.ftl +**/examples/**/.ftl +/.ftl + buildengine/.gitignore go.work* junit*.xml diff --git a/Justfile b/Justfile index 0f6d1eb3f3..50af62da26 100644 --- a/Justfile +++ b/Justfile @@ -9,7 +9,7 @@ KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar" RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip" TIMESTAMP := `date +%s` SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto" -ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" +ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/compile/main-work-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" FRONTEND_OUT := "frontend/dist/index.html" EXTENSION_OUT := "extensions/vscode/dist/extension.js" PROTOS_IN := "backend/protos/xyz/block/ftl/v1/schema/schema.proto backend/protos/xyz/block/ftl/v1/console/console.proto backend/protos/xyz/block/ftl/v1/ftl.proto backend/protos/xyz/block/ftl/v1/schema/runtime.proto" diff --git a/buildengine/build.go b/buildengine/build.go index 39756aa6b3..2b4fdd7ac9 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -22,11 +22,11 @@ const BuildLockTimeout = time.Minute // Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func Build(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { - return buildModule(ctx, sch, module, filesTransaction) +func Build(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { + return buildModule(ctx, projectRootDir, sch, module, filesTransaction) } -func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { +func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { release, err := flock.Acquire(ctx, filepath.Join(module.Config.Dir, ".ftl.lock"), BuildLockTimeout) if err != nil { return err @@ -43,7 +43,7 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTr logger.Infof("Building module") switch module.Config.Language { case "go": - err = buildGoModule(ctx, sch, module, filesTransaction) + err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction) case "kotlin": err = buildKotlinModule(ctx, sch, module) case "rust": diff --git a/buildengine/build_go.go b/buildengine/build_go.go index ee8fb1c98b..8eba01a747 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -8,8 +8,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGoModule(ctx context.Context, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { - if err := compile.Build(ctx, module.Config.Dir, sch, transaction); err != nil { +func buildGoModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { + if err := compile.Build(ctx, projectRootDir, module.Config.Dir, sch, transaction); err != nil { return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)} } return nil diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index 498e26e43b..cdfd9c8210 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -11,181 +11,6 @@ import ( "github.com/TBD54566975/ftl/backend/schema" ) -func TestGenerateGoModule(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - sch := &schema.Schema{ - Modules: []*schema.Module{ - schema.Builtins(), - {Name: "other", Decls: []schema.Decl{ - &schema.Enum{ - Comments: []string{"This is an enum.", "", "It has 3 variants."}, - Name: "Color", - Export: true, - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, - }, - }, - &schema.Enum{ - Name: "ColorInt", - Export: true, - Type: &schema.Int{}, - Variants: []*schema.EnumVariant{ - {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, - {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, - {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, - }, - }, - &schema.Enum{ - Comments: []string{"This is type enum."}, - Name: "TypeEnum", - Export: true, - Variants: []*schema.EnumVariant{ - {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, - {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, - }, - }, - &schema.Data{Name: "EchoRequest", Export: true}, - &schema.Data{ - Comments: []string{"This is an echo data response."}, - Name: "EchoResponse", Export: true}, - &schema.Verb{ - Name: "echo", - Export: true, - Request: &schema.Ref{Name: "EchoRequest"}, - Response: &schema.Ref{Name: "EchoResponse"}, - }, - &schema.Data{Name: "SinkReq", Export: true}, - &schema.Verb{ - Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, - Name: "sink", - Export: true, - Request: &schema.Ref{Name: "SinkReq"}, - Response: &schema.Unit{}, - }, - &schema.Data{Name: "SourceResp", Export: true}, - &schema.Verb{ - Name: "source", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Ref{Name: "SourceResp"}, - }, - &schema.Verb{ - Name: "nothing", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Unit{}, - }, - }}, - {Name: "test"}, - }, - } - expected := `// Code generated by FTL. DO NOT EDIT. - -package other - -import ( - "context" - - "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" -) - -var _ = context.Background - -// This is an enum. -// -// It has 3 variants. -// -//ftl:enum -type Color string -const ( - Red Color = "Red" - Blue Color = "Blue" - Green Color = "Green" -) - -//ftl:enum -type ColorInt int -const ( - RedInt ColorInt = 0 - BlueInt ColorInt = 1 - GreenInt ColorInt = 2 -) - -// This is type enum. -// -//ftl:enum -type TypeEnum interface { typeEnum() } - -type A int - -func (A) typeEnum() {} - -type B string - -func (B) typeEnum() {} - -type EchoRequest struct { -} - -// This is an echo data response. -// -type EchoResponse struct { -} - -//ftl:verb -func Echo(context.Context, EchoRequest) (EchoResponse, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} - -type SinkReq struct { -} - -// This is a sink verb. -// -// Here is another line for this comment! -// -//ftl:verb -func Sink(context.Context, SinkReq) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") -} - -type SourceResp struct { -} - -//ftl:verb -func Source(context.Context) (SourceResp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") -} - -//ftl:verb -func Nothing(context.Context) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") -} - -func init() { - reflection.Register( - reflection.SumType[TypeEnum]( - *new(A), - *new(B), - ), - ) -} -` - bctx := buildContext{ - moduleDir: "testdata/another", - buildDir: "_ftl", - sch: sch, - } - testBuild(t, bctx, "", []assertion{ - assertGeneratedModule("go/modules/other/external_module.go", expected), - }) -} - func TestGoBuildClearsBuildDir(t *testing.T) { if testing.Short() { t.SkipNow() @@ -198,82 +23,22 @@ func TestGoBuildClearsBuildDir(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/another", - buildDir: "_ftl", + buildDir: ".ftl", sch: sch, } testBuildClearsBuildDir(t, bctx) } -func TestMetadataImportsExcluded(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - sch := &schema.Schema{ - Modules: []*schema.Module{ - schema.Builtins(), - {Name: "test", Decls: []schema.Decl{ - &schema.Data{ - Comments: []string{"Request data type."}, - Name: "Req", Export: true}, - &schema.Data{Name: "Resp", Export: true}, - &schema.Verb{ - Comments: []string{"This is a verb."}, - Name: "call", - Export: true, - Request: &schema.Ref{Name: "Req"}, - Response: &schema.Ref{Name: "Resp"}, - Metadata: []schema.Metadata{ - &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, - }, - }, - }}, - }, - } - expected := `// Code generated by FTL. DO NOT EDIT. - -package test - -import ( - "context" -) - -var _ = context.Background - -// Request data type. -// -type Req struct { -} - -type Resp struct { -} - -// This is a verb. -// -//ftl:verb -func Call(context.Context, Req) (Resp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} -` - bctx := buildContext{ - moduleDir: "testdata/another", - buildDir: "_ftl", - sch: sch, - } - testBuild(t, bctx, "", []assertion{ - assertGeneratedModule("go/modules/test/external_module.go", expected), - }) -} - func TestExternalType(t *testing.T) { if testing.Short() { t.SkipNow() } bctx := buildContext{ moduleDir: "testdata/external", - buildDir: "_ftl", + buildDir: ".ftl", sch: &schema.Schema{}, } - testBuild(t, bctx, "unsupported external type", []assertion{ + testBuild(t, bctx, "", "unsupported external type", []assertion{ assertBuildProtoErrors( "unsupported external type \"time.Month\"", "unsupported type \"time.Month\" for field \"Month\"", @@ -302,10 +67,10 @@ func TestGoModVersion(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/highgoversion", - buildDir: "_ftl", + buildDir: ".ftl", sch: sch, } - testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), []assertion{}) + testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), "", []assertion{}) } func TestGeneratedTypeRegistry(t *testing.T) { @@ -345,10 +110,10 @@ func TestGeneratedTypeRegistry(t *testing.T) { assert.NoError(t, err) bctx := buildContext{ moduleDir: "testdata/other", - buildDir: "_ftl", + buildDir: ".ftl", sch: sch, } - testBuild(t, bctx, "", []assertion{ + testBuild(t, bctx, "", "", []assertion{ assertGeneratedMain(string(expected)), }) } diff --git a/buildengine/build_test.go b/buildengine/build_test.go index 83a05b6c09..e29e244ea2 100644 --- a/buildengine/build_test.go +++ b/buildengine/build_test.go @@ -38,6 +38,7 @@ func (t *mockModifyFilesTransaction) End() error { func testBuild( t *testing.T, bctx buildContext, + expectedGeneratStubsErrMsg string, // emptystr if no error expected expectedBuildErrMsg string, // emptystr if no error expected assertions []assertion, ) { @@ -47,12 +48,31 @@ func testBuild( assert.NoError(t, err, "Error getting absolute path for module directory") module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) - if len(expectedBuildErrMsg) > 0 { + + projectRootDir := t.TempDir() + + configs := []moduleconfig.ModuleConfig{} + if bctx.moduleDir != "" { + config, err := moduleconfig.LoadModuleConfig(bctx.moduleDir) + assert.NoError(t, err, "Error loading project config") + configs = append(configs, config) + } + + // generate stubs to create the shared modules directory + err = GenerateStubs(ctx, projectRootDir, bctx.sch.Modules, configs) + if len(expectedGeneratStubsErrMsg) > 0 { assert.Error(t, err) - assert.Contains(t, err.Error(), expectedBuildErrMsg) + assert.Contains(t, err.Error(), expectedGeneratStubsErrMsg) } else { assert.NoError(t, err) + + err = Build(ctx, projectRootDir, bctx.sch, module, &mockModifyFilesTransaction{}) + if len(expectedBuildErrMsg) > 0 { + assert.Error(t, err) + assert.Contains(t, err.Error(), expectedBuildErrMsg) + } else { + assert.NoError(t, err) + } } for _, a := range assertions { @@ -70,10 +90,16 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { abs, err := filepath.Abs(bctx.moduleDir) assert.NoError(t, err, "Error getting absolute path for module directory") + projectRoot := t.TempDir() + + // generate stubs to create the shared modules directory + err = GenerateStubs(ctx, projectRoot, bctx.sch.Modules, []moduleconfig.ModuleConfig{{Dir: bctx.moduleDir}}) + assert.NoError(t, err) + // build to generate the build directory module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // create a temporary file in the build directory @@ -85,7 +111,7 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { // build to clear the old build directory module, err = LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // ensure the temporary file was removed diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index 029f2d33af..3550668d26 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -10,6 +10,7 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/sha256" ) @@ -71,11 +72,17 @@ func TestDeploy(t *testing.T) { module, err := LoadModule(modulePath) assert.NoError(t, err) + projectRootDir := t.TempDir() + + // generate stubs to create the shared modules directory + err = GenerateStubs(ctx, projectRootDir, sch.Modules, []moduleconfig.ModuleConfig{module.Config}) + assert.NoError(t, err) + // Build first to make sure the files are there. - err = Build(ctx, sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, projectRootDir, sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) - sum, err := sha256.SumFile(modulePath + "/_ftl/main") + sum, err := sha256.SumFile(modulePath + "/.ftl/main") assert.NoError(t, err) client := &mockDeployClient{ diff --git a/buildengine/discover_test.go b/buildengine/discover_test.go index b63d7d3416..0aec6ccc97 100644 --- a/buildengine/discover_test.go +++ b/buildengine/discover_test.go @@ -22,7 +22,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "alpha", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -35,7 +35,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "another", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -48,7 +48,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle1", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -61,7 +61,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle2", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -100,7 +100,7 @@ func TestDiscoverModules(t *testing.T) { Deploy: []string{ "main", }, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{ @@ -140,7 +140,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "highgoversion", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -153,7 +153,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "other", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, diff --git a/buildengine/engine.go b/buildengine/engine.go index 03894be43d..806acc2d69 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -21,6 +21,7 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/rpc" "github.com/TBD54566975/ftl/internal/slices" @@ -65,6 +66,7 @@ type Listener interface { type Engine struct { client ftlv1connect.ControllerServiceClient moduleMetas *xsync.MapOf[string, moduleMeta] + projectRoot string moduleDirs []string watcher *Watcher controllerSchema *xsync.MapOf[string, *schema.Module] @@ -97,10 +99,11 @@ func WithListener(listener Listener) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, options ...Option) (*Engine, error) { +func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, projectRoot string, moduleDirs []string, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, + projectRoot: projectRoot, moduleDirs: moduleDirs, moduleMetas: xsync.NewMapOf[string, moduleMeta](), watcher: NewWatcher(), @@ -566,6 +569,21 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, } errCh := make(chan error, 1024) for _, group := range topology { + groupSchemas := map[string]*schema.Module{} + metas, err := e.gatherGroupSchemas(builtModules, group, groupSchemas) + if err != nil { + return err + } + + moduleConfigs := make([]moduleconfig.ModuleConfig, len(metas)) + for i, meta := range metas { + moduleConfigs[i] = meta.module.Config + } + err = GenerateStubs(ctx, e.projectRoot, maps.Values(groupSchemas), moduleConfigs) + if err != nil { + return err + } + // Collect schemas to be inserted into "built" map for subsequent groups. schemas := make(chan *schema.Module, len(group)) @@ -664,7 +682,7 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ if e.listener != nil { e.listener.OnBuildStarted(meta.module) } - err := Build(ctx, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) + err := Build(ctx, e.projectRoot, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) if err != nil { return err } @@ -677,6 +695,29 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ return nil } +// Construct a combined schema for a group of modules and their transitive dependencies. +func (e *Engine) gatherGroupSchemas( + moduleSchemas map[string]*schema.Module, + group []string, + out map[string]*schema.Module, +) ([]moduleMeta, error) { + var metas []moduleMeta + for _, module := range group { + if module == "builtin" { + continue // Skip the builtin module + } + + meta, ok := e.moduleMetas.Load(module) + if ok { + metas = append(metas, meta) + if err := e.gatherSchemas(moduleSchemas, meta.module, out); err != nil { + return nil, err + } + } + } + return metas, nil +} + // Construct a combined schema for a module and its transitive dependencies. func (e *Engine) gatherSchemas( moduleSchemas map[string]*schema.Module, diff --git a/buildengine/engine_test.go b/buildengine/engine_test.go index 73a5211349..a44ab8768a 100644 --- a/buildengine/engine_test.go +++ b/buildengine/engine_test.go @@ -16,7 +16,7 @@ func TestEngine(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/alpha", "testdata/other", "testdata/another"}) + engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/alpha", "testdata/other", "testdata/another"}) assert.NoError(t, err) defer engine.Close() @@ -64,7 +64,7 @@ func TestCycleDetection(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/depcycle1", "testdata/depcycle2"}) + engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/depcycle1", "testdata/depcycle2"}) assert.NoError(t, err) defer engine.Close() diff --git a/buildengine/stubs.go b/buildengine/stubs.go new file mode 100644 index 0000000000..911d00d104 --- /dev/null +++ b/buildengine/stubs.go @@ -0,0 +1,26 @@ +package buildengine + +import ( + "context" + "fmt" + + "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" + "github.com/TBD54566975/ftl/go-runtime/compile" +) + +// GenerateStubs generates stubs for the given modules. +// +// Currently, only Go stubs are supported. Kotlin and other language stubs can be added in the future. +func GenerateStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { + return generateGoStubs(ctx, projectRoot, modules, moduleConfigs) +} + +func generateGoStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { + sch := &schema.Schema{Modules: modules} + err := compile.GenerateStubsForModules(ctx, projectRoot, moduleConfigs, sch) + if err != nil { + return fmt.Errorf("failed to generate Go stubs: %w", err) + } + return nil +} diff --git a/buildengine/stubs_test.go b/buildengine/stubs_test.go new file mode 100644 index 0000000000..c4c1efe5b5 --- /dev/null +++ b/buildengine/stubs_test.go @@ -0,0 +1,250 @@ +package buildengine + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" + "github.com/TBD54566975/ftl/internal/log" + "github.com/alecthomas/assert/v2" +) + +func TestGenerateGoStubs(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + modules := []*schema.Module{ + schema.Builtins(), + {Name: "other", Decls: []schema.Decl{ + &schema.Enum{ + Comments: []string{"This is an enum.", "", "It has 3 variants."}, + Name: "Color", + Export: true, + Type: &schema.String{}, + Variants: []*schema.EnumVariant{ + {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, + {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, + {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, + }, + }, + &schema.Enum{ + Name: "ColorInt", + Export: true, + Type: &schema.Int{}, + Variants: []*schema.EnumVariant{ + {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, + {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, + {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, + }, + }, + &schema.Enum{ + Comments: []string{"This is type enum."}, + Name: "TypeEnum", + Export: true, + Variants: []*schema.EnumVariant{ + {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, + {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, + }, + }, + &schema.Data{Name: "EchoRequest", Export: true}, + &schema.Data{ + Comments: []string{"This is an echo data response."}, + Name: "EchoResponse", Export: true}, + &schema.Verb{ + Name: "echo", + Export: true, + Request: &schema.Ref{Name: "EchoRequest"}, + Response: &schema.Ref{Name: "EchoResponse"}, + }, + &schema.Data{Name: "SinkReq", Export: true}, + &schema.Verb{ + Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, + Name: "sink", + Export: true, + Request: &schema.Ref{Name: "SinkReq"}, + Response: &schema.Unit{}, + }, + &schema.Data{Name: "SourceResp", Export: true}, + &schema.Verb{ + Name: "source", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Ref{Name: "SourceResp"}, + }, + &schema.Verb{ + Name: "nothing", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Unit{}, + }, + }}, + {Name: "test"}, + } + + expected := `// Code generated by FTL. DO NOT EDIT. + +package other + +import ( + "context" + + "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" +) + +var _ = context.Background + +// This is an enum. +// +// It has 3 variants. +// +//ftl:enum +type Color string +const ( + Red Color = "Red" + Blue Color = "Blue" + Green Color = "Green" +) + +//ftl:enum +type ColorInt int +const ( + RedInt ColorInt = 0 + BlueInt ColorInt = 1 + GreenInt ColorInt = 2 +) + +// This is type enum. +// +//ftl:enum +type TypeEnum interface { typeEnum() } + +type A int + +func (A) typeEnum() {} + +type B string + +func (B) typeEnum() {} + +type EchoRequest struct { +} + +// This is an echo data response. +// +type EchoResponse struct { +} + +//ftl:verb +func Echo(context.Context, EchoRequest) (EchoResponse, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} + +type SinkReq struct { +} + +// This is a sink verb. +// +// Here is another line for this comment! +// +//ftl:verb +func Sink(context.Context, SinkReq) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") +} + +type SourceResp struct { +} + +//ftl:verb +func Source(context.Context) (SourceResp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") +} + +//ftl:verb +func Nothing(context.Context) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") +} + +func init() { + reflection.Register( + reflection.SumType[TypeEnum]( + *new(A), + *new(B), + ), + ) +} +` + + ctx := log.ContextWithNewDefaultLogger(context.Background()) + projectRoot := t.TempDir() + err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) + assert.NoError(t, err) + + generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/other/external_module.go") + fileContent, err := os.ReadFile(generatedPath) + assert.NoError(t, err) + assert.Equal(t, expected, string(fileContent)) +} + +func TestMetadataImportsExcluded(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + modules := []*schema.Module{ + schema.Builtins(), + {Name: "test", Decls: []schema.Decl{ + &schema.Data{ + Comments: []string{"Request data type."}, + Name: "Req", Export: true}, + &schema.Data{Name: "Resp", Export: true}, + &schema.Verb{ + Comments: []string{"This is a verb."}, + Name: "call", + Export: true, + Request: &schema.Ref{Name: "Req"}, + Response: &schema.Ref{Name: "Resp"}, + Metadata: []schema.Metadata{ + &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, + }, + }, + }}, + } + + expected := `// Code generated by FTL. DO NOT EDIT. + +package test + +import ( + "context" +) + +var _ = context.Background + +// Request data type. +// +type Req struct { +} + +type Resp struct { +} + +// This is a verb. +// +//ftl:verb +func Call(context.Context, Req) (Resp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} +` + ctx := log.ContextWithNewDefaultLogger(context.Background()) + projectRoot := t.TempDir() + err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) + assert.NoError(t, err) + + generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/test/external_module.go") + fileContent, err := os.ReadFile(generatedPath) + assert.NoError(t, err) + assert.Equal(t, expected, string(fileContent)) +} diff --git a/cmd/ftl/cmd_box.go b/cmd/ftl/cmd_box.go index d09214c887..887eefb763 100644 --- a/cmd/ftl/cmd_box.go +++ b/cmd/ftl/cmd_box.go @@ -123,7 +123,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_box_run.go b/cmd/ftl/cmd_box_run.go index 0f9cb08b52..0565edd540 100644 --- a/cmd/ftl/cmd_box_run.go +++ b/cmd/ftl/cmd_box_run.go @@ -17,6 +17,7 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/buildengine" + "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" @@ -33,7 +34,7 @@ type boxRunCmd struct { ControllerTimeout time.Duration `help:"Timeout for Controller start." default:"30s"` } -func (b *boxRunCmd) Run(ctx context.Context) error { +func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { conn, err := databasetesting.CreateForDevel(ctx, b.DSN, b.Recreate) if err != nil { return fmt.Errorf("failed to create database: %w", err) @@ -74,7 +75,7 @@ func (b *boxRunCmd) Run(ctx context.Context) error { return fmt.Errorf("controller failed to start: %w", err) } - engine, err := buildengine.New(ctx, client, []string{b.Dir}) + engine, err := buildengine.New(ctx, client, projConfig.Root(), []string{b.Dir}) if err != nil { return fmt.Errorf("failed to create build engine: %w", err) } diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index 35e826e740..a3f13869b2 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -22,7 +22,7 @@ func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServic if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index 0ecd47a6ed..a50c7f25fc 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -5,6 +5,7 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/buildengine" + "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/rpc" ) @@ -15,9 +16,9 @@ type deployCmd struct { NoWait bool `help:"Do not wait for deployment to complete." default:"false"` } -func (d *deployCmd) Run(ctx context.Context) error { +func (d *deployCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go index 2741faecfe..e596d84fa3 100644 --- a/cmd/ftl/cmd_dev.go +++ b/cmd/ftl/cmd_dev.go @@ -89,7 +89,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error }) } - engine, err := buildengine.New(ctx, client, d.Dirs, opts...) + engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, opts...) if err != nil { return err } diff --git a/cmd/ftl/cmd_init.go b/cmd/ftl/cmd_init.go index b394639797..b2cafb64e3 100644 --- a/cmd/ftl/cmd_init.go +++ b/cmd/ftl/cmd_init.go @@ -86,7 +86,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { scanner := bufio.NewScanner(f) for scanner.Scan() { - if strings.TrimSpace(scanner.Text()) == "**/_ftl" { + if strings.TrimSpace(scanner.Text()) == "**/.ftl" { return nil } } @@ -96,7 +96,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { } // append if not already present - if _, err = f.WriteString("**/_ftl\n"); err != nil { + if _, err = f.WriteString("**/.ftl\n"); err != nil { return err } diff --git a/cmd/ftl/cmd_new.go b/cmd/ftl/cmd_new.go index 831364a03c..1afa32ad1f 100644 --- a/cmd/ftl/cmd_new.go +++ b/cmd/ftl/cmd_new.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "github.com/TBD54566975/scaffolder" @@ -30,9 +31,10 @@ type newCmd struct { } type newGoCmd struct { - Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` - Dir string `arg:"" help:"Directory to initialize the module in."` - Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` + Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` + Dir string `arg:"" help:"Directory to initialize the module in."` + Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` + GoVersion string } type newKotlinCmd struct { @@ -58,6 +60,8 @@ func (i newGoCmd) Run(ctx context.Context) error { logger := log.FromContext(ctx) logger.Debugf("Creating FTL Go module %q in %s", name, path) + + i.GoVersion = runtime.Version()[2:] if err := scaffold(ctx, config.Hermit, goruntime.Files(), i.Dir, i, scaffolder.Exclude("^go.mod$")); err != nil { return err } diff --git a/common/moduleconfig/moduleconfig.go b/common/moduleconfig/moduleconfig.go index dddad51340..990e96b732 100644 --- a/common/moduleconfig/moduleconfig.go +++ b/common/moduleconfig/moduleconfig.go @@ -133,7 +133,7 @@ func setConfigDefaults(moduleDir string, config *ModuleConfig) error { case "go": if config.DeployDir == "" { - config.DeployDir = "_ftl" + config.DeployDir = ".ftl" } if len(config.Deploy) == 0 { config.Deploy = []string{"main"} diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index dbc72146ec..a539cf6ee4 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "ftl", - "version": "0.1.2", + "version": "0.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ftl", - "version": "0.1.2", + "version": "0.0.0", "dependencies": { "lookpath": "^1.2.2", "semver": "^7.6.0", diff --git a/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl b/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl similarity index 94% rename from go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl rename to go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl index 56b07af1f7..df0b6eca75 100644 --- a/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl +++ b/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go b/go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl similarity index 100% rename from go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go rename to go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl diff --git a/go-runtime/compile/build-template/go.work.tmpl b/go-runtime/compile/build-template/go.work.tmpl index 683cd1d340..cbd6143cf0 100644 --- a/go-runtime/compile/build-template/go.work.tmpl +++ b/go-runtime/compile/build-template/go.work.tmpl @@ -2,6 +2,8 @@ go {{ .GoVersion }} use ( . - _ftl/go/modules - _ftl/go/main +{{- range .SharedModulesPaths }} + {{ . }} +{{- end }} + .ftl/go/main ) diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index e1012a747d..4b37742949 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -3,7 +3,6 @@ package compile import ( "context" "fmt" - "maps" "os" "path" "path/filepath" @@ -15,7 +14,7 @@ import ( "unicode" sets "github.com/deckarep/golang-set/v2" - gomaps "golang.org/x/exp/maps" + "golang.org/x/exp/maps" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" @@ -35,12 +34,16 @@ import ( "github.com/TBD54566975/ftl/internal/reflect" ) +type MainWorkContext struct { + GoVersion string + SharedModulesPaths []string +} + type ExternalModuleContext struct { - ModuleDir string *schema.Schema GoVersion string FTLVersion string - Main string + Module *schema.Module Replacements []*modfile.Replace } @@ -53,13 +56,14 @@ type goVerb struct { } type mainModuleContext struct { - GoVersion string - FTLVersion string - Name string - Verbs []goVerb - Replacements []*modfile.Replace - SumTypes []goSumType - LocalSumTypes []goSumType + GoVersion string + FTLVersion string + Name string + SharedModulesPaths []string + Verbs []goVerb + Replacements []*modfile.Replace + SumTypes []goSumType + LocalSumTypes []goSumType } type goSumType struct { @@ -79,25 +83,14 @@ type ModifyFilesTransaction interface { End() error } -func (b ExternalModuleContext) NonMainModules() []*schema.Module { - modules := make([]*schema.Module, 0, len(b.Modules)) - for _, module := range b.Modules { - if module.Name == b.Main { - continue - } - modules = append(modules, module) - } - return modules -} - -const buildDirName = "_ftl" +const buildDirName = ".ftl" func buildDir(moduleDir string) string { return filepath.Join(moduleDir, buildDirName) } // Build the given module. -func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { +func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { if err := filesTransaction.Begin(); err != nil { return err } @@ -130,19 +123,27 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans funcs := maps.Clone(scaffoldFuncs) - logger.Debugf("Generating external modules") - if err := generateExternalModules(ExternalModuleContext{ - ModuleDir: moduleDir, - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Schema: sch, - Main: config.Module, - Replacements: replacements, - }); err != nil { - return fmt.Errorf("failed to generate external modules: %w", err) + buildDir := buildDir(moduleDir) + err = os.MkdirAll(buildDir, 0750) + if err != nil { + return fmt.Errorf("failed to create build directory: %w", err) + } + + var sharedModulesPaths []string + for _, mod := range sch.Modules { + if mod.Name == config.Module { + continue + } + sharedModulesPaths = append(sharedModulesPaths, filepath.Join(projectRootDir, buildDirName, "go", "modules", mod.Name)) + } + + if err := internal.ScaffoldZip(mainWorkTemplateFiles(), moduleDir, MainWorkContext{ + GoVersion: goModVersion, + SharedModulesPaths: sharedModulesPaths, + }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { + return fmt.Errorf("failed to scaffold zip: %w", err) } - buildDir := buildDir(moduleDir) logger.Debugf("Extracting schema") result, err := ExtractModuleSchema(config.Dir, sch) if err != nil { @@ -186,13 +187,14 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans goVerbs = append(goVerbs, goverb) } if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{ - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Name: result.Module.Name, - Verbs: goVerbs, - Replacements: replacements, - SumTypes: getSumTypes(result.Module, sch, result.NativeNames), - LocalSumTypes: getLocalSumTypes(result.Module), + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Name: result.Module.Name, + SharedModulesPaths: sharedModulesPaths, + Verbs: goVerbs, + Replacements: replacements, + SumTypes: getSumTypes(result.Module, sch, result.NativeNames), + LocalSumTypes: getLocalSumTypes(result.Module), }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -200,7 +202,7 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans logger.Debugf("Tidying go.mod files") wg, wgctx := errgroup.WithContext(ctx) wg.Go(func() error { - if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { + if err := exec.Command(wgctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", moduleDir, err) } return filesTransaction.ModifiedFiles(filepath.Join(moduleDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) @@ -212,13 +214,6 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans } return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) - modulesDir := filepath.Join(buildDir, "go", "modules") - wg.Go(func() error { - if err := exec.Command(wgctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { - return fmt.Errorf("%s: failed to tidy go.mod: %w", modulesDir, err) - } - return filesTransaction.ModifiedFiles(filepath.Join(modulesDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) - }) if err := wg.Wait(); err != nil { return err } @@ -227,10 +222,16 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } -func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error { - goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod")) +func GenerateStubsForModules(ctx context.Context, projectRoot string, moduleConfigs []moduleconfig.ModuleConfig, sch *schema.Schema) error { + logger := log.FromContext(ctx) + logger.Debugf("Generating stubs for modules") + + sharedFtlDir := filepath.Join(projectRoot, buildDirName) + + // Wipe the modules directory to ensure we don't have any stale modules. + err := os.RemoveAll(sharedFtlDir) if err != nil { - return fmt.Errorf("failed to propagate replacements for library %q: %w", dir, err) + return fmt.Errorf("failed to remove %s: %w", sharedFtlDir, err) } ftlVersion := "" @@ -238,24 +239,65 @@ func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *sc ftlVersion = ftl.Version } - return generateExternalModules(ExternalModuleContext{ - ModuleDir: dir, - GoVersion: goModFile.Go.Version, - FTLVersion: ftlVersion, - Schema: schema, - Replacements: replacements, - }) -} + for _, module := range sch.Modules { + var moduleConfig *moduleconfig.ModuleConfig + for _, mc := range moduleConfigs { + mcCopy := mc + if mc.Module == module.Name { + moduleConfig = &mcCopy + break + } + } -func generateExternalModules(context ExternalModuleContext) error { - // Wipe the modules directory to ensure we don't have any stale modules. - err := os.RemoveAll(filepath.Join(buildDir(context.ModuleDir), "go", "modules")) - if err != nil { - return err + var goModVersion string + var replacements []*modfile.Replace + + // If there's no module config, use the go.mod file for the first config we find. + if moduleConfig == nil { + if len(moduleConfigs) > 0 { + _, goModVersion, err = updateGoModule(filepath.Join(moduleConfigs[0].Dir, "go.mod")) + if err != nil { + return err + } + } else { + // The best we can do here if we don't have a module to read from is to use the current Go version. + goModVersion = runtime.Version()[2:] + } + + replacements = []*modfile.Replace{} + } else { + replacements, goModVersion, err = updateGoModule(filepath.Join(moduleConfig.Dir, "go.mod")) + if err != nil { + return err + } + } + + goVersion := runtime.Version()[2:] + if semver.Compare("v"+goVersion, "v"+goModVersion) < 0 { + return fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion) + } + + context := ExternalModuleContext{ + Schema: sch, + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Module: module, + Replacements: replacements, + } + + funcs := maps.Clone(scaffoldFuncs) + err = internal.ScaffoldZip(externalModuleTemplateFiles(), projectRoot, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) + if err != nil { + return fmt.Errorf("failed to scaffold zip: %w", err) + } + + modulesDir := filepath.Join(sharedFtlDir, "go", "modules", module.Name) + if err := exec.Command(ctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { + return fmt.Errorf("failed to tidy go.mod: %w", err) + } } - funcs := maps.Clone(scaffoldFuncs) - return internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) + return nil } var scaffoldFuncs = scaffolder.FuncMap{ @@ -571,7 +613,7 @@ func getLocalSumTypes(module *schema.Module) []goSumType { } } } - out := gomaps.Values(sumTypes) + out := maps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) @@ -614,7 +656,7 @@ func getSumTypes(module *schema.Module, sch *schema.Schema, nativeNames NativeNa Variants: variants, } } - out := gomaps.Values(sumTypes) + out := maps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) diff --git a/go-runtime/compile/devel.go b/go-runtime/compile/devel.go index c6f2f6270c..345309da66 100644 --- a/go-runtime/compile/devel.go +++ b/go-runtime/compile/devel.go @@ -8,9 +8,14 @@ import ( "github.com/TBD54566975/ftl/internal" ) +func mainWorkTemplateFiles() *zip.Reader { + return internal.ZipRelativeToCaller("main-work-template") +} + func externalModuleTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("external-module-template") } + func buildTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("build-template") } diff --git a/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl similarity index 66% rename from go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl rename to go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl index a7c7c1b2de..657e8f20ab 100644 --- a/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl +++ b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl @@ -1,13 +1,13 @@ // Code generated by FTL. DO NOT EDIT. -package {{.Name}} +package {{.Module.Name}} import ( "context" -{{- range $import, $alias := (.|imports)}} +{{- range $import, $alias := (.Module | imports)}} {{if $alias}}{{$alias}} {{end}}"{{$import}}" {{- end}} -{{- $sumTypes := $ | sumTypes}} +{{- $sumTypes := (.Module | sumTypes)}} {{- if $sumTypes}} "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" @@ -16,18 +16,18 @@ import ( var _ = context.Background -{{- range .Decls }} +{{- range .Module.Decls }} {{- if .IsExported}} {{if .Comments}} {{.Comments|comment -}} // {{- end}} {{- if is "Topic" .}} -var {{.Name|title}} = ftl.Topic[{{type $ .Event}}]("{{.Name}}") +var {{.Name|title}} = ftl.Topic[{{type $.Module .Event}}]("{{.Name}}") {{- else if and (is "Enum" .) .IsValueEnum}} {{- $enumName := .Name}} //ftl:enum -type {{.Name|title}} {{type $ .Type}} +type {{.Name|title}} {{type $.Module .Type}} const ( {{- range .Variants }} {{.Name|title}} {{$enumName}} = {{.Value|value}} @@ -38,14 +38,14 @@ const ( {{$enumInterfaceFuncName := enumInterfaceFunc . -}} type {{.Name|title}} interface { {{$enumInterfaceFuncName}}() } {{- range .Variants }} -{{if (or (basicType $ .) (isStandaloneEnumVariant .))}} -type {{.Name|title}} {{type $ .Value.Value}} +{{if (or (basicType $.Module .) (isStandaloneEnumVariant .))}} +type {{.Name|title}} {{type $.Module .Value.Value}} {{end}} func ({{.Name|title}}) {{$enumInterfaceFuncName}}() {} {{- end}} {{- else if is "TypeAlias" .}} //ftl:typealias -type {{.Name|title}} {{type $ .Type}} +type {{.Name|title}} {{type $.Module .Type}} {{- else if is "Data" .}} type {{.Name|title}} {{- if .TypeParameters}}[ @@ -54,25 +54,25 @@ type {{.Name|title}} {{- end -}} ]{{- end}} struct { {{- range .Fields}} - {{.Name|title}} {{type $ .Type}} `json:"{{.Name}}"` + {{.Name|title}} {{type $.Module .Type}} `json:"{{.Name}}"` {{- end}} } {{- else if is "Verb" .}} //ftl:verb -{{- if and (eq (type $ .Request) "ftl.Unit") (eq (type $ .Response) "ftl.Unit")}} +{{- if and (eq (type $.Module .Request) "ftl.Unit") (eq (type $.Module .Response) "ftl.Unit")}} func {{.Name|title}}(context.Context) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") } -{{- else if eq (type $ .Request) "ftl.Unit"}} -func {{.Name|title}}(context.Context) ({{type $ .Response}}, error) { +{{- else if eq (type $.Module .Request) "ftl.Unit"}} +func {{.Name|title}}(context.Context) ({{type $.Module .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") } -{{- else if eq (type $ .Response) "ftl.Unit"}} -func {{.Name|title}}(context.Context, {{type $ .Request}}) error { +{{- else if eq (type $.Module .Response) "ftl.Unit"}} +func {{.Name|title}}(context.Context, {{type $.Module .Request}}) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") } {{- else}} -func {{.Name|title}}(context.Context, {{type $ .Request}}) ({{type $ .Response}}, error) { +func {{.Name|title}}(context.Context, {{type $.Module .Request}}) ({{type $.Module .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") } {{- end}} diff --git a/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl similarity index 80% rename from go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl rename to go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl index 2f1d05e47d..be3d760791 100644 --- a/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl +++ b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl @@ -1,4 +1,4 @@ -module ftl +module ftl/{{ .Module.Name }} go {{ .GoVersion }} @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/go-runtime/compile/external-module-template/go.mod b/go-runtime/compile/external-module-template/go.mod deleted file mode 100644 index 8da40da65a..0000000000 --- a/go-runtime/compile/external-module-template/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module exclude - -go 1.22.2 diff --git a/go-runtime/compile/external-module-template/go.work.tmpl b/go-runtime/compile/external-module-template/go.work.tmpl deleted file mode 100644 index 204b39b25e..0000000000 --- a/go-runtime/compile/external-module-template/go.work.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -go {{ .GoVersion }} - -use ( - . - _ftl/go/modules -) diff --git a/go-runtime/compile/main-work-template/go.work.tmpl b/go-runtime/compile/main-work-template/go.work.tmpl new file mode 100644 index 0000000000..1dbed607a3 --- /dev/null +++ b/go-runtime/compile/main-work-template/go.work.tmpl @@ -0,0 +1,8 @@ +go {{ .GoVersion }} + +use ( + . +{{- range .SharedModulesPaths }} + {{ . }} +{{- end }} +) diff --git a/go-runtime/compile/release.go b/go-runtime/compile/release.go index 8b909688cd..a258dd2170 100644 --- a/go-runtime/compile/release.go +++ b/go-runtime/compile/release.go @@ -8,12 +8,23 @@ import ( _ "embed" ) +//go:embed main-work-template.zip +var mainWorkTemplateBytes []byte + //go:embed external-module-template.zip var externalModuleTemplateBytes []byte //go:embed build-template.zip var buildTemplateBytes []byte +func mainWorkTemplateFiles() *zip.Reader { + zr, err := zip.NewReader(bytes.NewReader(mainWorkTemplateBytes), int64(len(mainWorkTemplateBytes))) + if err != nil { + panic(err) + } + return zr +} + func externalModuleTemplateFiles() *zip.Reader { zr, err := zip.NewReader(bytes.NewReader(externalModuleTemplateBytes), int64(len(externalModuleTemplateBytes))) if err != nil { diff --git a/go-runtime/compile/schema_test.go b/go-runtime/compile/schema_test.go index 8c974259bf..abbc0d83db 100644 --- a/go-runtime/compile/schema_test.go +++ b/go-runtime/compile/schema_test.go @@ -56,7 +56,7 @@ func TestExtractModuleSchema(t *testing.T) { assert.NoError(t, err) actual := schema.Normalise(r.Module) expected := `module one { - config configValue one.Config + config configValue one.Config secret secretValue String database postgres testDb @@ -184,6 +184,9 @@ func TestExtractModuleSchemaTwo(t *testing.T) { if testing.Short() { t.SkipNow() } + + assert.NoError(t, prebuildTestModule(t, "testdata/two")) + r, err := ExtractModuleSchema("testdata/two", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, r.Errors, nil) @@ -365,6 +368,9 @@ func TestExtractModulePubSub(t *testing.T) { if testing.Short() { t.SkipNow() } + + assert.NoError(t, prebuildTestModule(t, "testdata/pubsub")) + r, err := ExtractModuleSchema("testdata/pubsub", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, nil, r.Errors, "expected no schema errors") diff --git a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl index 372814bd3b..7d6b63f1d2 100644 --- a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl +++ b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl @@ -1,9 +1,9 @@ module ftl/{{ .Name }} -go 1.21 +go {{ .GoVersion }} require github.com/TBD54566975/ftl latest {{- range $old, $new := .Replace }} replace {{ $old }} => {{ $new }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/integration/harness.go b/integration/harness.go index d4c55fc955..6cb0af3369 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -17,6 +17,7 @@ import ( "connectrpc.com/connect" "github.com/alecthomas/assert/v2" "github.com/alecthomas/types/optional" + "github.com/otiai10/copy" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" @@ -73,11 +74,23 @@ func run(t *testing.T, ftlConfigPath string, startController bool, actions ...Ac assert.True(t, ok) if ftlConfigPath != "" { - // Use a path into the testdata directory instead of one relative to - // tmpDir. Otherwise we have a chicken and egg situation where the config - // can't be loaded until the module is copied over, and the config itself - // is used by FTL during startup. - t.Setenv("FTL_CONFIG", filepath.Join(cwd, "testdata", "go", ftlConfigPath)) + ftlConfigPath = filepath.Join(cwd, "testdata", "go", ftlConfigPath) + projectPath := filepath.Join(tmpDir, "ftl-project.toml") + + // Copy the specified FTL config to the temporary directory. + err = copy.Copy(ftlConfigPath, projectPath) + if err == nil { + t.Setenv("FTL_CONFIG", projectPath) + } else { + // Use a path into the testdata directory instead of one relative to + // tmpDir. Otherwise we have a chicken and egg situation where the config + // can't be loaded until the module is copied over, and the config itself + // is used by FTL during startup. + // Some tests still rely on this behavior, so we can't remove it entirely. + t.Logf("Failed to copy %s to %s: %s", ftlConfigPath, projectPath, err) + t.Setenv("FTL_CONFIG", ftlConfigPath) + } + } else { err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644) assert.NoError(t, err) From 27a844d3398265353007bdbd39c10c815a60dfe7 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Thu, 4 Jul 2024 20:45:50 +1000 Subject: [PATCH 12/23] fix: split migrations since 0.268.3 out into separate files (#1975) Use `dbmate new ` to create a new migration file. --- .../{001_init.sql => 20231103205514_init.sql} | 10 ---------- .../schema/20240704103403_create_module_secrets.sql | 13 +++++++++++++ bin/hermit.hcl | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-) rename backend/controller/sql/schema/{001_init.sql => 20231103205514_init.sql} (98%) create mode 100644 backend/controller/sql/schema/20240704103403_create_module_secrets.sql diff --git a/backend/controller/sql/schema/001_init.sql b/backend/controller/sql/schema/20231103205514_init.sql similarity index 98% rename from backend/controller/sql/schema/001_init.sql rename to backend/controller/sql/schema/20231103205514_init.sql index fb5c20462c..68b34021b2 100644 --- a/backend/controller/sql/schema/001_init.sql +++ b/backend/controller/sql/schema/20231103205514_init.sql @@ -518,14 +518,4 @@ CREATE TABLE module_configuration UNIQUE (module, name) ); -CREATE TABLE module_secrets -( - id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - created_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'), - module TEXT, -- If NULL, configuration is global. - name TEXT NOT NULL, - url TEXT NOT NULL, - UNIQUE (module, name) -); - -- migrate:down diff --git a/backend/controller/sql/schema/20240704103403_create_module_secrets.sql b/backend/controller/sql/schema/20240704103403_create_module_secrets.sql new file mode 100644 index 0000000000..ce861ed185 --- /dev/null +++ b/backend/controller/sql/schema/20240704103403_create_module_secrets.sql @@ -0,0 +1,13 @@ +-- migrate:up + +CREATE TABLE module_secrets +( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'), + module TEXT, -- If NULL, configuration is global. + name TEXT NOT NULL, + url TEXT NOT NULL, + UNIQUE (module, name) +); + +-- migrate:down diff --git a/bin/hermit.hcl b/bin/hermit.hcl index 64e90f9240..02334feb23 100644 --- a/bin/hermit.hcl +++ b/bin/hermit.hcl @@ -1,9 +1,9 @@ env = { + "DBMATE_MIGRATIONS_DIR": "${HERMIT_ENV}/backend/controller/sql/schema", "FTL_ENDPOINT": "http://localhost:8892", + "FTL_INIT_GO_REPLACE": "github.com/TBD54566975/ftl=${HERMIT_ENV}", "FTL_SOURCE": "${HERMIT_ENV}", "OTEL_METRIC_EXPORT_INTERVAL": "5000", "PATH": "${HERMIT_ENV}/scripts:${HERMIT_ENV}/frontend/node_modules/.bin:${HERMIT_ENV}/extensions/vscode/node_modules/.bin:${PATH}", - "FTL_INIT_GO_REPLACE": "github.com/TBD54566975/ftl=${HERMIT_ENV}", - } sources = ["env:///bin/packages", "https://github.com/cashapp/hermit-packages.git"] From 43a3b1ee2b74ae8afeeb49777759133549cdb8a4 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Fri, 5 Jul 2024 05:46:25 +1000 Subject: [PATCH 13/23] feat: add /debug/pprof handlers (#1976) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds debugging pprof HTTP handlers. The controller serves this directly, while the runner forwards to the user verb server's handlers, allowing verb servers to be debugged directly. Here's an example of CPU profiling the echo module: ``` 🐚 ~/dev/ftl $ go tool pprof http://localhost:8894/debug/pprof/profile Fetching profile over HTTP from http://localhost:8894/debug/pprof/profile Saved profile in /Users/alec/pprof/pprof.main.samples.cpu.001.pb.gz File: main Type: cpu Time: Jul 5, 2024 at 5:33am (AEST) Duration: 30.18s, Total samples = 600ms ( 1.99%) Entering interactive mode (type "help" for commands, "o" for options) (pprof) top10 Showing nodes accounting for 600ms, 100% of 600ms total Showing top 10 nodes out of 73 flat flat% sum% cum cum% 210ms 35.00% 35.00% 210ms 35.00% runtime.pthread_cond_signal 190ms 31.67% 66.67% 190ms 31.67% runtime.kevent 110ms 18.33% 85.00% 110ms 18.33% runtime.pthread_cond_wait 50ms 8.33% 93.33% 50ms 8.33% syscall.syscall 10ms 1.67% 95.00% 10ms 1.67% connectrpc.com/connect.grpcErrorToTrailer 10ms 1.67% 96.67% 10ms 1.67% runtime.scanobject 10ms 1.67% 98.33% 10ms 1.67% runtime.usleep 10ms 1.67% 100% 10ms 1.67% runtime.write1 0 0% 100% 30ms 5.00% bufio.(*Writer).Flush 0 0% 100% 10ms 1.67% connectrpc.com/connect.(*Handler).ServeHTTP ``` pprof index looks like this: image --- backend/controller/controller.go | 1 + backend/runner/runner.go | 14 ++++++++++++++ common/plugin/serve.go | 2 ++ internal/http/pprof.go | 24 ++++++++++++++++++++++++ internal/rpc/server.go | 9 +++++++++ 5 files changed, 50 insertions(+) create mode 100644 internal/http/pprof.go diff --git a/backend/controller/controller.go b/backend/controller/controller.go index ca2f47ad3b..896d373ce5 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -149,6 +149,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali rpc.GRPC(ftlv1connect.NewAdminServiceHandler, admin), rpc.GRPC(pbconsoleconnect.NewConsoleServiceHandler, console), rpc.HTTP("/", consoleHandler), + rpc.PProf(), ) }) diff --git a/backend/runner/runner.go b/backend/runner/runner.go index 0b9bf62833..dc0754d809 100644 --- a/backend/runner/runner.go +++ b/backend/runner/runner.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "math/rand" + "net/http" + "net/http/httputil" "net/url" "os" "path/filepath" @@ -108,6 +110,7 @@ func Start(ctx context.Context, config Config) error { return rpc.Serve(ctx, config.Bind, rpc.GRPC(ftlv1connect.NewVerbServiceHandler, svc), rpc.GRPC(ftlv1connect.NewRunnerServiceHandler, svc), + rpc.HTTP("/", svc), ) } @@ -353,6 +356,17 @@ func (s *Service) Terminate(ctx context.Context, c *connect.Request[ftlv1.Termin }), nil } +func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { + deployment, ok := s.deployment.Load().Get() + if !ok { + http.Error(w, "no deployment", http.StatusNotFound) + return + } + proxy := httputil.NewSingleHostReverseProxy(deployment.plugin.Endpoint) + proxy.ServeHTTP(w, r) + +} + func (s *Service) makeDeployment(ctx context.Context, key model.DeploymentKey, plugin *plugin.Plugin[ftlv1connect.VerbServiceClient]) *deployment { return &deployment{ ctx: ctx, diff --git a/common/plugin/serve.go b/common/plugin/serve.go index 1b1d99b12f..cb3dedfc5b 100644 --- a/common/plugin/serve.go +++ b/common/plugin/serve.go @@ -21,6 +21,7 @@ import ( "golang.org/x/net/http2/h2c" _ "github.com/TBD54566975/ftl/internal/automaxprocs" // Set GOMAXPROCS to match Linux container CPU quota. + ftlhttp "github.com/TBD54566975/ftl/internal/http" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/rpc" ) @@ -154,6 +155,7 @@ func Start[Impl any, Iface any, Config any]( reflector := grpcreflect.NewStaticReflector(servicePaths...) mux.Handle(grpcreflect.NewHandlerV1(reflector)) mux.Handle(grpcreflect.NewHandlerV1Alpha(reflector)) + ftlhttp.RegisterPprof(mux) // Start the server. http1Server := &http.Server{ diff --git a/internal/http/pprof.go b/internal/http/pprof.go new file mode 100644 index 0000000000..8c48c19f4f --- /dev/null +++ b/internal/http/pprof.go @@ -0,0 +1,24 @@ +package http + +import ( + "net/http" + "net/http/pprof" +) + +// RegisterPprof registers all pprof handlers and the index on the provided ServeMux. +func RegisterPprof(mux *http.ServeMux) { + mux.HandleFunc("/debug/pprof", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/debug/pprof/", http.StatusFound) + }) + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + mux.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("/debug/pprof/heap", pprof.Handler("heap")) + mux.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) + mux.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) + mux.Handle("/debug/pprof/block", pprof.Handler("block")) + mux.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) +} diff --git a/internal/rpc/server.go b/internal/rpc/server.go index abf7737b1c..7036fb692b 100644 --- a/internal/rpc/server.go +++ b/internal/rpc/server.go @@ -9,6 +9,8 @@ import ( "strings" "time" + gaphttp "github.com/TBD54566975/ftl/internal/http" + "connectrpc.com/connect" "connectrpc.com/grpcreflect" "github.com/alecthomas/concurrency" @@ -40,6 +42,13 @@ func GRPC[Iface, Impl Pingable](constructor GRPCServerConstructor[Iface], impl I } } +// PProf adds /debug/pprof routes to the server. +func PProf() Option { + return func(so *serverOptions) { + gaphttp.RegisterPprof(so.mux) + } +} + // RawGRPC is a convenience function for registering a GRPC server with default options without Pingable. func RawGRPC[Iface, Impl any](constructor RawGRPCServerConstructor[Iface], impl Impl, options ...connect.HandlerOption) Option { return func(o *serverOptions) { From ce2a3ae116ee3652390cc67dbc0dfa10d1fc6924 Mon Sep 17 00:00:00 2001 From: gak Date: Fri, 5 Jul 2024 10:26:41 +1000 Subject: [PATCH 14/23] feat: suggest using project file in ctx in tests (#1977) .. when failing to access config or secrets Fixes #1874 --------- Co-authored-by: Alec Thomas --- go-runtime/ftl/ftltest/fake.go | 6 ++-- go-runtime/ftl/ftltest/ftltest_test.go | 39 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 go-runtime/ftl/ftltest/ftltest_test.go diff --git a/go-runtime/ftl/ftltest/fake.go b/go-runtime/ftl/ftltest/fake.go index efd35b323f..7aa50342ae 100644 --- a/go-runtime/ftl/ftltest/fake.go +++ b/go-runtime/ftl/ftltest/fake.go @@ -90,7 +90,7 @@ func (f *fakeFTL) setConfig(name string, value any) error { func (f *fakeFTL) GetConfig(ctx context.Context, name string, dest any) error { data, ok := f.configValues[name] if !ok { - return fmt.Errorf("secret value %q not found: %w", name, configuration.ErrNotFound) + return fmt.Errorf("secret value %q not found, did you remember to ctx := ftltest.Context(ftltest.WithDefaultProjectFile()) ?: %w", name, configuration.ErrNotFound) } return json.Unmarshal(data, dest) } @@ -107,7 +107,7 @@ func (f *fakeFTL) setSecret(name string, value any) error { func (f *fakeFTL) GetSecret(ctx context.Context, name string, dest any) error { data, ok := f.secretValues[name] if !ok { - return fmt.Errorf("config value %q not found: %w", name, configuration.ErrNotFound) + return fmt.Errorf("config value %q not found, did you remember to ctx := ftltest.Context(ftltest.WithDefaultProjectFile()) ?: %w", name, configuration.ErrNotFound) } return json.Unmarshal(data, dest) } @@ -140,7 +140,7 @@ func (f *fakeFTL) CallMap(ctx context.Context, mapper any, value any, mapImpl fu if f.allowMapCalls { return actuallyCallMap(ctx, mapImpl) } - panic("map calls not allowed in tests by default. ftltest.Context should be instantiated with either ftltest.WithMapsAllowed() or a mock for the specific map being called using ftltest.WhenMap(...)") + panic("map calls not allowed in tests by default, ftltest.Context should be instantiated with either ftltest.WithMapsAllowed() or a mock for the specific map being called using ftltest.WhenMap(...)") } func makeMapKey(mapper any) uintptr { diff --git a/go-runtime/ftl/ftltest/ftltest_test.go b/go-runtime/ftl/ftltest/ftltest_test.go new file mode 100644 index 0000000000..41d192a596 --- /dev/null +++ b/go-runtime/ftl/ftltest/ftltest_test.go @@ -0,0 +1,39 @@ +package ftltest + +import ( + "context" + "fmt" + "testing" + + "github.com/TBD54566975/ftl/go-runtime/ftl" + "github.com/TBD54566975/ftl/go-runtime/internal" + "github.com/TBD54566975/ftl/internal/log" + "github.com/alecthomas/assert/v2" +) + +func PanicsWithErr(t testing.TB, substr string, fn func()) { + t.Helper() + defer func() { + err := recover() + if err == nil { + t.Fatal("Expected panic, but got nil") + } + + errStr := fmt.Sprintf("%v", err) + assert.Contains(t, errStr, substr, "Expected panic message to contain %q, but got %q", substr, errStr) + }() + fn() +} + +func TestFtlTestProjectNotLoadedInContext(t *testing.T) { + ctx := log.ContextWithNewDefaultLogger(context.Background()) + ctx = internal.WithContext(ctx, newFakeFTL(ctx)) + + // This should panic suggesting to use ftltest.WithDefaultProjectFile() + PanicsWithErr(t, "ftltest.WithDefaultProjectFile()", func() { + _ = ftl.Secret[string]("moo").Get(ctx) + }) + PanicsWithErr(t, "ftltest.WithDefaultProjectFile()", func() { + _ = ftl.Config[string]("moo").Get(ctx) + }) +} From 409c4ab7f84507a50d4b4d1cf3563748a8ec33e9 Mon Sep 17 00:00:00 2001 From: gak Date: Fri, 5 Jul 2024 10:27:03 +1000 Subject: [PATCH 15/23] fix: console was crashing quietly, relative paths (#1979) - Now log why console crashed - Use gitRoot based path for running console --- backend/controller/controller.go | 2 +- cmd/ftl/cmd_serve.go | 1 + frontend/local.go | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/controller/controller.go b/backend/controller/controller.go index 896d373ce5..66811b0c87 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -113,7 +113,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali } else { consoleHandler, err = frontend.Server(ctx, config.ContentTime, config.Bind, config.ConsoleURL) if err != nil { - return err + return fmt.Errorf("could not start console: %w", err) } logger.Infof("Web console available at: %s", config.Bind) } diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go index 10a7a10442..05573c14db 100644 --- a/cmd/ftl/cmd_serve.go +++ b/cmd/ftl/cmd_serve.go @@ -134,6 +134,7 @@ func (s *serveCmd) run(ctx context.Context, projConfig projectconfig.Config, ini wg.Go(func() error { if err := controller.Start(controllerCtx, config, runnerScaling, dal); err != nil { + logger.Errorf(err, "controller%d failed: %v", i, err) return fmt.Errorf("controller%d failed: %w", i, err) } return nil diff --git a/frontend/local.go b/frontend/local.go index 66f8b45940..6ea258410a 100644 --- a/frontend/local.go +++ b/frontend/local.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "path" "time" "github.com/TBD54566975/ftl/internal" @@ -33,7 +34,7 @@ func Server(ctx context.Context, timestamp time.Time, publicURL *url.URL, allowO return nil, err } - err = exec.Command(ctx, log.Debug, "frontend", "npm", "run", "dev").Start() + err = exec.Command(ctx, log.Debug, path.Join(gitRoot, "frontend"), "npm", "run", "dev").Start() if err != nil { return nil, err } From bf4539feb8958b51bded3d0be6ae4361c00a3062 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Fri, 5 Jul 2024 10:55:30 +1000 Subject: [PATCH 16/23] =?UTF-8?q?revert:=20"fix:=20validate=20configs=20an?= =?UTF-8?q?d=20secrets=20against=20the=20schema=20before=20s=E2=80=A6=20(#?= =?UTF-8?q?1978)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …etting (#1858)" This reverts commit f4cb64adaae99fd2ab1739a0d84c2393f794ce9e. # Conflicts: # backend/controller/admin/testdata/go/ftl-project-dr.toml --- backend/controller/admin/admin.go | 115 ++------ backend/controller/admin/admin_test.go | 130 +-------- backend/controller/admin/local_client.go | 43 +-- backend/controller/admin/local_client_test.go | 65 ----- .../admin/testdata/go/dischema/dischema.go | 23 -- .../admin/testdata/go/dischema/ftl.toml | 2 - .../admin/testdata/go/dischema/go.mod | 43 --- .../admin/testdata/go/dischema/go.sum | 142 ---------- .../admin/testdata/go/ftl-project-dr.toml | 9 - backend/controller/controller.go | 18 +- backend/controller/dal/dal.go | 17 -- backend/controller/ingress/ingress.go | 203 +++++++++++++- backend/controller/ingress/ingress_test.go | 4 +- backend/controller/ingress/request.go | 42 ++- backend/schema/jsonvalidate.go | 251 ------------------ 15 files changed, 279 insertions(+), 828 deletions(-) delete mode 100644 backend/controller/admin/local_client_test.go delete mode 100644 backend/controller/admin/testdata/go/dischema/dischema.go delete mode 100644 backend/controller/admin/testdata/go/dischema/ftl.toml delete mode 100644 backend/controller/admin/testdata/go/dischema/go.mod delete mode 100644 backend/controller/admin/testdata/go/dischema/go.sum delete mode 100644 backend/controller/admin/testdata/go/ftl-project-dr.toml delete mode 100644 backend/schema/jsonvalidate.go diff --git a/backend/controller/admin/admin.go b/backend/controller/admin/admin.go index 02eb45a853..3ca0e6ae75 100644 --- a/backend/controller/admin/admin.go +++ b/backend/controller/admin/admin.go @@ -9,29 +9,20 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" - "github.com/TBD54566975/ftl/backend/schema" cf "github.com/TBD54566975/ftl/common/configuration" - "github.com/TBD54566975/ftl/go-runtime/encoding" - "github.com/TBD54566975/ftl/internal/log" ) type AdminService struct { - schr SchemaRetriever - cm *cf.Manager[cf.Configuration] - sm *cf.Manager[cf.Secrets] + cm *cf.Manager[cf.Configuration] + sm *cf.Manager[cf.Secrets] } var _ ftlv1connect.AdminServiceHandler = (*AdminService)(nil) -type SchemaRetriever interface { - GetActiveSchema(ctx context.Context) (*schema.Schema, error) -} - -func NewAdminService(cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets], schr SchemaRetriever) *AdminService { +func NewAdminService(cm *cf.Manager[cf.Configuration], sm *cf.Manager[cf.Secrets]) *AdminService { return &AdminService{ - schr: schr, - cm: cm, - sm: sm, + cm: cm, + sm: sm, } } @@ -43,7 +34,7 @@ func (s *AdminService) Ping(ctx context.Context, req *connect.Request[ftlv1.Ping func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv1.ListConfigRequest]) (*connect.Response[ftlv1.ListConfigResponse], error) { listing, err := s.cm.List(ctx) if err != nil { - return nil, fmt.Errorf("failed to list configs: %w", err) + return nil, err } configs := []*ftlv1.ListConfigResponse_Config{} @@ -82,13 +73,13 @@ func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv // ConfigGet returns the configuration value for a given ref string. func (s *AdminService) ConfigGet(ctx context.Context, req *connect.Request[ftlv1.GetConfigRequest]) (*connect.Response[ftlv1.GetConfigResponse], error) { var value any - err := s.cm.Get(ctx, refFromConfigRef(req.Msg.GetRef()), &value) + err := s.cm.Get(ctx, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), &value) if err != nil { - return nil, fmt.Errorf("failed to get from config manager: %w", err) + return nil, err } vb, err := json.MarshalIndent(value, "", " ") if err != nil { - return nil, fmt.Errorf("failed to marshal value: %w", err) + return nil, err } return connect.NewResponse(&ftlv1.GetConfigResponse{Value: vb}), nil } @@ -110,15 +101,10 @@ func configProviderKey(p *ftlv1.ConfigProvider) string { // ConfigSet sets the configuration at the given ref to the provided value. func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1.SetConfigRequest]) (*connect.Response[ftlv1.SetConfigResponse], error) { - err := s.validateAgainstSchema(ctx, false, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value) - if err != nil { - return nil, err - } - pkey := configProviderKey(req.Msg.Provider) - err = s.cm.SetJSON(ctx, pkey, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value) + err := s.cm.SetJSON(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), req.Msg.Value) if err != nil { - return nil, fmt.Errorf("failed to set config: %w", err) + return nil, err } return connect.NewResponse(&ftlv1.SetConfigResponse{}), nil } @@ -126,9 +112,9 @@ func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1 // ConfigUnset unsets the config value at the given ref. func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error) { pkey := configProviderKey(req.Msg.Provider) - err := s.cm.Unset(ctx, pkey, refFromConfigRef(req.Msg.GetRef())) + err := s.cm.Unset(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name)) if err != nil { - return nil, fmt.Errorf("failed to unset config: %w", err) + return nil, err } return connect.NewResponse(&ftlv1.UnsetConfigResponse{}), nil } @@ -137,7 +123,7 @@ func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftl func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftlv1.ListSecretsRequest]) (*connect.Response[ftlv1.ListSecretsResponse], error) { listing, err := s.sm.List(ctx) if err != nil { - return nil, fmt.Errorf("failed to list secrets: %w", err) + return nil, err } secrets := []*ftlv1.ListSecretsResponse_Secret{} for _, secret := range listing { @@ -172,13 +158,13 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl // SecretGet returns the secret value for a given ref string. func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1.GetSecretRequest]) (*connect.Response[ftlv1.GetSecretResponse], error) { var value any - err := s.sm.Get(ctx, refFromConfigRef(req.Msg.GetRef()), &value) + err := s.sm.Get(ctx, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), &value) if err != nil { - return nil, fmt.Errorf("failed to get from secret manager: %w", err) + return nil, err } vb, err := json.MarshalIndent(value, "", " ") if err != nil { - return nil, fmt.Errorf("failed to marshal value: %w", err) + return nil, err } return connect.NewResponse(&ftlv1.GetSecretResponse{Value: vb}), nil } @@ -204,15 +190,10 @@ func secretProviderKey(p *ftlv1.SecretProvider) string { // SecretSet sets the secret at the given ref to the provided value. func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error) { - err := s.validateAgainstSchema(ctx, true, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value) - if err != nil { - return nil, err - } - pkey := secretProviderKey(req.Msg.Provider) - err = s.sm.SetJSON(ctx, pkey, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value) + err := s.sm.SetJSON(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name), req.Msg.Value) if err != nil { - return nil, fmt.Errorf("failed to set secret: %w", err) + return nil, err } return connect.NewResponse(&ftlv1.SetSecretResponse{}), nil } @@ -220,63 +201,9 @@ func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1 // SecretUnset unsets the secret value at the given ref. func (s *AdminService) SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error) { pkey := secretProviderKey(req.Msg.Provider) - err := s.sm.Unset(ctx, pkey, refFromConfigRef(req.Msg.GetRef())) + err := s.sm.Unset(ctx, pkey, cf.NewRef(*req.Msg.Ref.Module, req.Msg.Ref.Name)) if err != nil { - return nil, fmt.Errorf("failed to unset secret: %w", err) + return nil, err } return connect.NewResponse(&ftlv1.UnsetSecretResponse{}), nil } - -func refFromConfigRef(cr *ftlv1.ConfigRef) cf.Ref { - return cf.NewRef(cr.GetModule(), cr.GetName()) -} - -func (s *AdminService) validateAgainstSchema(ctx context.Context, isSecret bool, ref cf.Ref, value json.RawMessage) error { - logger := log.FromContext(ctx) - - // Globals aren't in the module schemas, so we have nothing to validate against. - if !ref.Module.Ok() { - return nil - } - - // If we can't retrieve an active schema, skip validation. - sch, err := s.schr.GetActiveSchema(ctx) - if err != nil { - logger.Debugf("skipping validation; could not get the active schema: %v", err) - return nil - } - - r := schema.RefKey{Module: ref.Module.Default(""), Name: ref.Name}.ToRef() - decl, ok := sch.Resolve(r).Get() - if !ok { - return fmt.Errorf("declaration %q not found", ref.Name) - } - - var fieldType schema.Type - if isSecret { - decl, ok := decl.(*schema.Secret) - if !ok { - return fmt.Errorf("%q is not a secret declaration", ref.Name) - } - fieldType = decl.Type - } else { - decl, ok := decl.(*schema.Config) - if !ok { - return fmt.Errorf("%q is not a config declaration", ref.Name) - } - fieldType = decl.Type - } - - var v any - err = encoding.Unmarshal(value, &v) - if err != nil { - return fmt.Errorf("could not unmarshal JSON value: %w", err) - } - - err = schema.ValidateJSONValue(fieldType, []string{ref.Name}, v, sch) - if err != nil { - return fmt.Errorf("JSON validation failed: %w", err) - } - - return nil -} diff --git a/backend/controller/admin/admin_test.go b/backend/controller/admin/admin_test.go index 636d46044f..a0add822bf 100644 --- a/backend/controller/admin/admin_test.go +++ b/backend/controller/admin/admin_test.go @@ -13,7 +13,6 @@ import ( "github.com/alecthomas/types/optional" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" - "github.com/TBD54566975/ftl/backend/schema" cf "github.com/TBD54566975/ftl/common/configuration" "github.com/TBD54566975/ftl/internal/log" ) @@ -32,7 +31,7 @@ func TestAdminService(t *testing.T) { cf.InlineProvider[cf.Secrets]{}, }) assert.NoError(t, err) - admin := NewAdminService(cm, sm, &diskSchemaRetriever{}) + admin := NewAdminService(cm, sm) assert.NotZero(t, admin) expectedEnvarValue, err := json.MarshalIndent(map[string]string{"bar": "barfoo"}, "", " ") @@ -141,130 +140,3 @@ func testAdminSecrets( assert.Equal(t, entry.Value, string(resp.Msg.Value)) } } - -var testSchema = schema.MustValidate(&schema.Schema{ - Modules: []*schema.Module{ - { - Name: "batmobile", - Comments: []string{"A batmobile comment"}, - Decls: []schema.Decl{ - &schema.Secret{ - Comments: []string{"top secret"}, - Name: "owner", - Type: &schema.String{}, - }, - &schema.Secret{ - Comments: []string{"ultra secret"}, - Name: "horsepower", - Type: &schema.Int{}, - }, - &schema.Config{ - Comments: []string{"car color"}, - Name: "color", - Type: &schema.Ref{Module: "batmobile", Name: "Color"}, - }, - &schema.Config{ - Comments: []string{"car capacity"}, - Name: "capacity", - Type: &schema.Ref{Module: "batmobile", Name: "Capacity"}, - }, - &schema.Enum{ - Comments: []string{"Car colors"}, - Name: "Color", - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Black", Value: &schema.StringValue{Value: "Black"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, - }, - }, - &schema.Enum{ - Comments: []string{"Car capacities"}, - Name: "Capacity", - Type: &schema.Int{}, - Variants: []*schema.EnumVariant{ - {Name: "One", Value: &schema.IntValue{Value: int(1)}}, - {Name: "Two", Value: &schema.IntValue{Value: int(2)}}, - {Name: "Four", Value: &schema.IntValue{Value: int(4)}}, - }, - }, - }, - }, - }, -}) - -type mockSchemaRetriever struct { -} - -func (d *mockSchemaRetriever) GetActiveSchema(ctx context.Context) (*schema.Schema, error) { - return testSchema, nil -} - -func TestAdminValidation(t *testing.T) { - config := tempConfigPath(t, "testdata/ftl-project.toml", "admin") - ctx := log.ContextWithNewDefaultLogger(context.Background()) - - cm, err := cf.NewConfigurationManager(ctx, cf.ProjectConfigResolver[cf.Configuration]{Config: config}) - assert.NoError(t, err) - - sm, err := cf.New(ctx, - cf.ProjectConfigResolver[cf.Secrets]{Config: config}, - []cf.Provider[cf.Secrets]{ - cf.EnvarProvider[cf.Secrets]{}, - cf.InlineProvider[cf.Secrets]{}, - }) - assert.NoError(t, err) - admin := NewAdminService(cm, sm, &mockSchemaRetriever{}) - assert.NotZero(t, admin) - - testSetConfig(t, ctx, admin, "batmobile", "color", "Black", "") - testSetConfig(t, ctx, admin, "batmobile", "color", "Red", "JSON validation failed: Red is not a valid variant of enum batmobile.Color") - testSetConfig(t, ctx, admin, "batmobile", "capacity", 2, "") - testSetConfig(t, ctx, admin, "batmobile", "capacity", 3, "JSON validation failed: %!s(float64=3) is not a valid variant of enum batmobile.Capacity") - - testSetSecret(t, ctx, admin, "batmobile", "owner", "Bruce Wayne", "") - testSetSecret(t, ctx, admin, "batmobile", "owner", 99, "JSON validation failed: owner has wrong type, expected String found float64") - testSetSecret(t, ctx, admin, "batmobile", "horsepower", 1000, "") - testSetSecret(t, ctx, admin, "batmobile", "horsepower", "thousand", "JSON validation failed: horsepower has wrong type, expected Int found string") - - testSetConfig(t, ctx, admin, "", "city", "Gotham", "") - testSetSecret(t, ctx, admin, "", "universe", "DC", "") -} - -// nolint -func testSetConfig(t testing.TB, ctx context.Context, admin *AdminService, module string, name string, jsonVal any, expectedError string) { - t.Helper() - buffer, err := json.Marshal(jsonVal) - assert.NoError(t, err) - - configRef := &ftlv1.ConfigRef{Name: name} - if module != "" { - configRef.Module = &module - } - - _, err = admin.ConfigSet(ctx, connect.NewRequest(&ftlv1.SetConfigRequest{ - Provider: ftlv1.ConfigProvider_CONFIG_INLINE.Enum(), - Ref: configRef, - Value: buffer, - })) - assert.EqualError(t, err, expectedError) -} - -// nolint -func testSetSecret(t testing.TB, ctx context.Context, admin *AdminService, module string, name string, jsonVal any, expectedError string) { - t.Helper() - buffer, err := json.Marshal(jsonVal) - assert.NoError(t, err) - - configRef := &ftlv1.ConfigRef{Name: name} - if module != "" { - configRef.Module = &module - } - - _, err = admin.SecretSet(ctx, connect.NewRequest(&ftlv1.SetSecretRequest{ - Provider: ftlv1.SecretProvider_SECRET_INLINE.Enum(), - Ref: configRef, - Value: buffer, - })) - assert.EqualError(t, err, expectedError) -} diff --git a/backend/controller/admin/local_client.go b/backend/controller/admin/local_client.go index ad87330a29..73c659b0fb 100644 --- a/backend/controller/admin/local_client.go +++ b/backend/controller/admin/local_client.go @@ -2,14 +2,8 @@ package admin import ( "context" - "fmt" - "path/filepath" - "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/buildengine" "github.com/TBD54566975/ftl/common/configuration" - "github.com/TBD54566975/ftl/common/projectconfig" - "github.com/alecthomas/types/optional" ) // localClient reads and writes to local projectconfig files without making any network @@ -19,43 +13,8 @@ type localClient struct { *AdminService } -type diskSchemaRetriever struct { - // Omit to use the project root as the deploy root. - deployRoot optional.Option[string] -} - func newLocalClient(ctx context.Context) *localClient { cm := configuration.ConfigFromContext(ctx) sm := configuration.SecretsFromContext(ctx) - return &localClient{NewAdminService(cm, sm, &diskSchemaRetriever{})} -} - -func (s *diskSchemaRetriever) GetActiveSchema(ctx context.Context) (*schema.Schema, error) { - path, ok := projectconfig.DefaultConfigPath().Get() - if !ok { - return nil, fmt.Errorf("no project config path available") - } - projConfig, err := projectconfig.Load(ctx, path) - if err != nil { - return nil, fmt.Errorf("could not load project config: %w", err) - } - modules, err := buildengine.DiscoverModules(ctx, projConfig.AbsModuleDirs()) - if err != nil { - return nil, fmt.Errorf("could not discover modules: %w", err) - } - - sch := &schema.Schema{} - for _, m := range modules { - schemaPath := m.Config.Abs().Schema - if r, ok := s.deployRoot.Get(); ok { - schemaPath = filepath.Join(r, m.Config.Module, m.Config.DeployDir, m.Config.Schema) - } - - module, err := schema.ModuleFromProtoFile(schemaPath) - if err != nil { - return nil, fmt.Errorf("could not load module schema: %w", err) - } - sch.Upsert(module) - } - return sch, nil + return &localClient{NewAdminService(cm, sm)} } diff --git a/backend/controller/admin/local_client_test.go b/backend/controller/admin/local_client_test.go deleted file mode 100644 index 1d4f9eab4e..0000000000 --- a/backend/controller/admin/local_client_test.go +++ /dev/null @@ -1,65 +0,0 @@ -//go:build integration - -package admin - -import ( - "context" - "testing" - - cf "github.com/TBD54566975/ftl/common/configuration" - in "github.com/TBD54566975/ftl/integration" - "github.com/TBD54566975/ftl/internal/log" - "github.com/alecthomas/assert/v2" - "github.com/alecthomas/types/optional" -) - -func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) { - in.RunWithoutController(t, "ftl-project-dr.toml", - in.CopyModule("dischema"), - in.Build("dischema"), - func(t testing.TB, ic in.TestContext) { - dsr := &diskSchemaRetriever{deployRoot: optional.Some[string](ic.WorkingDir())} - sch, err := dsr.GetActiveSchema(ic.Context) - assert.NoError(t, err) - - module, ok := sch.Module("dischema").Get() - assert.Equal(t, ok, true) - assert.Equal(t, "dischema", module.Name) - }, - ) -} - -func TestDiskSchemaRetrieverWithNoSchema(t *testing.T) { - in.RunWithoutController(t, "ftl-project-dr.toml", - in.CopyModule("dischema"), - func(t testing.TB, ic in.TestContext) { - dsr := &diskSchemaRetriever{} - _, err := dsr.GetActiveSchema(ic.Context) - assert.Error(t, err) - }, - ) -} - -func TestAdminNoValidationWithNoSchema(t *testing.T) { - config := tempConfigPath(t, "testdata/ftl-project.toml", "admin") - ctx := log.ContextWithNewDefaultLogger(context.Background()) - - cm, err := cf.NewConfigurationManager(ctx, cf.ProjectConfigResolver[cf.Configuration]{Config: config}) - assert.NoError(t, err) - - sm, err := cf.New(ctx, - cf.ProjectConfigResolver[cf.Secrets]{Config: config}, - []cf.Provider[cf.Secrets]{ - cf.EnvarProvider[cf.Secrets]{}, - cf.InlineProvider[cf.Secrets]{}, - }) - assert.NoError(t, err) - - dsr := &diskSchemaRetriever{deployRoot: optional.Some(string(t.TempDir()))} - _, err = dsr.GetActiveSchema(ctx) - assert.Error(t, err) - - admin := NewAdminService(cm, sm, dsr) - testSetConfig(t, ctx, admin, "batmobile", "color", "Red", "") - testSetSecret(t, ctx, admin, "batmobile", "owner", 99, "") -} diff --git a/backend/controller/admin/testdata/go/dischema/dischema.go b/backend/controller/admin/testdata/go/dischema/dischema.go deleted file mode 100644 index f780f6a518..0000000000 --- a/backend/controller/admin/testdata/go/dischema/dischema.go +++ /dev/null @@ -1,23 +0,0 @@ -package dischema - -import ( - "context" - "fmt" - - "github.com/TBD54566975/ftl/go-runtime/ftl" // Import the FTL SDK. -) - -var defaultUser = ftl.Config[string]("defaultUser") - -type EchoRequest struct { - Name ftl.Option[string] `json:"name"` -} - -type EchoResponse struct { - Message string `json:"message"` -} - -//ftl:verb -func Echo(ctx context.Context, req EchoRequest) (EchoResponse, error) { - return EchoResponse{Message: fmt.Sprintf("Hello, %s!", req.Name.Default(defaultUser.Get(ctx)))}, nil -} diff --git a/backend/controller/admin/testdata/go/dischema/ftl.toml b/backend/controller/admin/testdata/go/dischema/ftl.toml deleted file mode 100644 index 19ad570a97..0000000000 --- a/backend/controller/admin/testdata/go/dischema/ftl.toml +++ /dev/null @@ -1,2 +0,0 @@ -module = "dischema" -language = "go" diff --git a/backend/controller/admin/testdata/go/dischema/go.mod b/backend/controller/admin/testdata/go/dischema/go.mod deleted file mode 100644 index 60aa2617ca..0000000000 --- a/backend/controller/admin/testdata/go/dischema/go.mod +++ /dev/null @@ -1,43 +0,0 @@ -module ftl/dischema - -go 1.22.2 - -toolchain go1.22.4 - -require github.com/TBD54566975/ftl v1.1.5 - -require ( - connectrpc.com/connect v1.16.1 // indirect - connectrpc.com/grpcreflect v1.2.0 // indirect - connectrpc.com/otelconnect v0.7.0 // indirect - github.com/alecthomas/concurrency v0.0.2 // indirect - github.com/alecthomas/participle/v2 v2.1.1 // indirect - github.com/alecthomas/types v0.16.0 // indirect - github.com/alessio/shellescape v1.4.2 // indirect - github.com/danieljoos/wincred v1.2.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/jpillora/backoff v1.0.0 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/swaggest/jsonschema-go v0.3.70 // indirect - github.com/swaggest/refl v1.3.0 // indirect - github.com/zalando/go-keyring v0.2.4 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.27.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect -) diff --git a/backend/controller/admin/testdata/go/dischema/go.sum b/backend/controller/admin/testdata/go/dischema/go.sum deleted file mode 100644 index e2d61f4b4e..0000000000 --- a/backend/controller/admin/testdata/go/dischema/go.sum +++ /dev/null @@ -1,142 +0,0 @@ -connectrpc.com/connect v1.16.1 h1:rOdrK/RTI/7TVnn3JsVxt3n028MlTRwmK5Q4heSpjis= -connectrpc.com/connect v1.16.1/go.mod h1:XpZAduBQUySsb4/KO5JffORVkDI4B6/EYPi7N8xpNZw= -connectrpc.com/grpcreflect v1.2.0 h1:Q6og1S7HinmtbEuBvARLNwYmTbhEGRpHDhqrPNlmK+U= -connectrpc.com/grpcreflect v1.2.0/go.mod h1:nwSOKmE8nU5u/CidgHtPYk1PFI3U9ignz7iDMxOYkSY= -connectrpc.com/otelconnect v0.7.0 h1:ZH55ZZtcJOTKWWLy3qmL4Pam4RzRWBJFOqTPyAqCXkY= -connectrpc.com/otelconnect v0.7.0/go.mod h1:Bt2ivBymHZHqxvo4HkJ0EwHuUzQN6k2l0oH+mp/8nwc= -github.com/TBD54566975/ftl v1.1.5 h1:PZq322WiBWUFnOHv51bnDjT/SbztEUVkEAHyGJA1HWY= -github.com/TBD54566975/ftl v1.1.5/go.mod h1:Z64GLoDe603sxe7Nytpx6Le/bB/Jlk8m52MVz+JKXtM= -github.com/TBD54566975/scaffolder v1.0.0 h1:QUFSy2wVzumLDg7IHcKC6AP+IYyqWe9Wxiu72nZn5qU= -github.com/TBD54566975/scaffolder v1.0.0/go.mod h1:auVpczIbOAdIhYDVSruIw41DanxOKB9bSvjf6MEl7Fs= -github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= -github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= -github.com/alecthomas/concurrency v0.0.2 h1:Q3kGPtLbleMbH9lHX5OBFvJygfyFw29bXZKBg+IEVuo= -github.com/alecthomas/concurrency v0.0.2/go.mod h1:GmuQb/iHX7mbNtPlC/WDzEFxDMB0HYFer2Qda9QTs7w= -github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8= -github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= -github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= -github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alecthomas/types v0.16.0 h1:o9+JSwCRB6DDaWDeR/Mg7v/zh3R+MlknM6DrnDyY7U0= -github.com/alecthomas/types v0.16.0/go.mod h1:Tswm0qQpjpVq8rn70OquRsUtFxbQKub/8TMyYYGI0+k= -github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= -github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/bool64/dev v0.2.34 h1:P9n315P8LdpxusnYQ0X7MP1CZXwBK5ae5RZrd+GdSZE= -github.com/bool64/dev v0.2.34/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= -github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= -github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= -github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= -github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= -github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= -github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= -github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/swaggest/assertjson v1.9.0 h1:dKu0BfJkIxv/xe//mkCrK5yZbs79jL7OVf9Ija7o2xQ= -github.com/swaggest/assertjson v1.9.0/go.mod h1:b+ZKX2VRiUjxfUIal0HDN85W0nHPAYUbYH5WkkSsFsU= -github.com/swaggest/jsonschema-go v0.3.70 h1:8Vx5nm5t/6DBFw2+WC0/Vp1ZVe9/4mpuA0tuAe0wwCI= -github.com/swaggest/jsonschema-go v0.3.70/go.mod h1:7N43/CwdaWgPUDfYV70K7Qm79tRqe/al7gLSt9YeGIE= -github.com/swaggest/refl v1.3.0 h1:PEUWIku+ZznYfsoyheF97ypSduvMApYyGkYF3nabS0I= -github.com/swaggest/refl v1.3.0/go.mod h1:3Ujvbmh1pfSbDYjC6JGG7nMgPvpG0ehQL4iNonnLNbg= -github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/zalando/go-keyring v0.2.4 h1:wi2xxTqdiwMKbM6TWwi+uJCG/Tum2UV0jqaQhCa9/68= -github.com/zalando/go-keyring v0.2.4/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= -go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= -modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= -modernc.org/libc v1.50.9 h1:hIWf1uz55lorXQhfoEoezdUHjxzuO6ceshET/yWjSjk= -modernc.org/libc v1.50.9/go.mod h1:15P6ublJ9FJR8YQCGy8DeQ2Uwur7iW9Hserr/T3OFZE= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/sqlite v1.30.0 h1:8YhPUs/HTnlEgErn/jSYQTwHN/ex8CjHHjg+K9iG7LM= -modernc.org/sqlite v1.30.0/go.mod h1:cgkTARJ9ugeXSNaLBPK3CqbOe7Ec7ZhWPoMFGldEYEw= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/backend/controller/admin/testdata/go/ftl-project-dr.toml b/backend/controller/admin/testdata/go/ftl-project-dr.toml deleted file mode 100644 index a65b02f7ff..0000000000 --- a/backend/controller/admin/testdata/go/ftl-project-dr.toml +++ /dev/null @@ -1,9 +0,0 @@ -name = "dr" -ftl-min-version = "dev" -hermit = false -no-git = false - -[global] - -[commands] - startup = ["echo disk retreiver"] diff --git a/backend/controller/controller.go b/backend/controller/controller.go index 66811b0c87..0881645a26 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -127,7 +127,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali cm := cf.ConfigFromContext(ctx) sm := cf.SecretsFromContext(ctx) - admin := admin.NewAdminService(cm, sm, dal) + admin := admin.NewAdminService(cm, sm) console := NewConsoleService(dal) ingressHandler := http.Handler(svc) @@ -282,7 +282,7 @@ func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - sch, err := s.dal.GetActiveSchema(r.Context()) + sch, err := s.getActiveSchema(r.Context()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -905,7 +905,7 @@ func (s *Service) callWithRequest( return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("body is required")) } - sch, err := s.dal.GetActiveSchema(ctx) + sch, err := s.getActiveSchema(ctx) if err != nil { return nil, err } @@ -1658,6 +1658,18 @@ func (s *Service) syncSchema(ctx context.Context) { } } +func (s *Service) getActiveSchema(ctx context.Context) (*schema.Schema, error) { + deployments, err := s.dal.GetActiveDeployments(ctx) + if err != nil { + return nil, err + } + return schema.ValidateSchema(&schema.Schema{ + Modules: slices.Map(deployments, func(d dal.Deployment) *schema.Module { + return d.Schema + }), + }) +} + func extractIngressRoutingEntries(req *ftlv1.CreateDeploymentRequest) []dal.IngressRoutingEntry { var ingressRoutes []dal.IngressRoutingEntry for _, decl := range req.Schema.Decls { diff --git a/backend/controller/dal/dal.go b/backend/controller/dal/dal.go index 968140315e..dcc3e70e3a 100644 --- a/backend/controller/dal/dal.go +++ b/backend/controller/dal/dal.go @@ -841,23 +841,6 @@ func (d *DAL) GetActiveDeployments(ctx context.Context) ([]Deployment, error) { }) } -// GetActiveSchema returns the schema for all active deployments. -func (d *DAL) GetActiveSchema(ctx context.Context) (*schema.Schema, error) { - deployments, err := d.GetActiveDeployments(ctx) - if err != nil { - return nil, err - } - sch, err := schema.ValidateSchema(&schema.Schema{ - Modules: slices.Map(deployments, func(d Deployment) *schema.Module { - return d.Schema - }), - }) - if err != nil { - return nil, fmt.Errorf("could not validate schema: %w", err) - } - return sch, nil -} - func (d *DAL) GetDeploymentsWithMinReplicas(ctx context.Context) ([]Deployment, error) { rows, err := d.db.GetDeploymentsWithMinReplicas(ctx) if err != nil { diff --git a/backend/controller/ingress/ingress.go b/backend/controller/ingress/ingress.go index fbcafd7192..afa896058a 100644 --- a/backend/controller/ingress/ingress.go +++ b/backend/controller/ingress/ingress.go @@ -1,10 +1,14 @@ package ingress import ( + "encoding/base64" "encoding/json" "fmt" "math/rand" + "reflect" + "strconv" "strings" + "time" "github.com/TBD54566975/ftl/backend/controller/dal" dalerrs "github.com/TBD54566975/ftl/backend/dal" @@ -12,6 +16,12 @@ import ( "github.com/TBD54566975/ftl/internal/slices" ) +type path []string + +func (p path) String() string { + return strings.TrimLeft(strings.Join(p, ""), ".") +} + func GetIngressRoute(routes []dal.IngressRoute, method string, path string) (*dal.IngressRoute, error) { var matchedRoutes = slices.Filter(routes, func(route dal.IngressRoute) bool { return matchSegments(route.Path, path, func(segment, value string) {}) @@ -56,11 +66,7 @@ func ValidateCallBody(body []byte, verb *schema.Verb, sch *schema.Schema) error return fmt.Errorf("HTTP request body is not valid JSON: %w", err) } - err = schema.ValidateJSONValue(verb.Request, []string{verb.Request.String()}, requestMap, sch) - if err != nil { - return fmt.Errorf("could not validate HTTP request body: %w", err) - } - return nil + return validateValue(verb.Request, []string{verb.Request.String()}, requestMap, sch) } func getBodyField(ref *schema.Ref, sch *schema.Schema) (*schema.Field, error) { @@ -82,3 +88,190 @@ func getBodyField(ref *schema.Ref, sch *schema.Schema) (*schema.Field, error) { return bodyField, nil } + +func validateValue(fieldType schema.Type, path path, value any, sch *schema.Schema) error { //nolint:maintidx + var typeMatches bool + switch fieldType := fieldType.(type) { + case *schema.Any: + typeMatches = true + + case *schema.Unit: + // TODO: Use type assertions consistently in this function rather than reflection. + rv := reflect.ValueOf(value) + if rv.Kind() != reflect.Map || rv.Len() != 0 { + return fmt.Errorf("%s must be an empty map", path) + } + return nil + + case *schema.Time: + str, ok := value.(string) + if !ok { + return fmt.Errorf("time %s must be an RFC3339 formatted string", path) + } + _, err := time.Parse(time.RFC3339Nano, str) + if err != nil { + return fmt.Errorf("time %s must be an RFC3339 formatted string: %w", path, err) + } + return nil + + case *schema.Int: + switch value := value.(type) { + case int64, float64: + typeMatches = true + case string: + if _, err := strconv.ParseInt(value, 10, 64); err == nil { + typeMatches = true + } + } + + case *schema.Float: + switch value := value.(type) { + case float64: + typeMatches = true + case string: + if _, err := strconv.ParseFloat(value, 64); err == nil { + typeMatches = true + } + } + + case *schema.String: + _, typeMatches = value.(string) + + case *schema.Bool: + switch value := value.(type) { + case bool: + typeMatches = true + case string: + if _, err := strconv.ParseBool(value); err == nil { + typeMatches = true + } + } + + case *schema.Array: + rv := reflect.ValueOf(value) + if rv.Kind() != reflect.Slice { + return fmt.Errorf("%s is not a slice", path) + } + elementType := fieldType.Element + for i := range rv.Len() { + elemPath := append(path, fmt.Sprintf("[%d]", i)) //nolint:gocritic + elem := rv.Index(i).Interface() + if err := validateValue(elementType, elemPath, elem, sch); err != nil { + return err + } + } + typeMatches = true + + case *schema.Map: + rv := reflect.ValueOf(value) + if rv.Kind() != reflect.Map { + return fmt.Errorf("%s is not a map", path) + } + keyType := fieldType.Key + valueType := fieldType.Value + for _, key := range rv.MapKeys() { + elemPath := append(path, fmt.Sprintf("[%q]", key)) //nolint:gocritic + elem := rv.MapIndex(key).Interface() + if err := validateValue(keyType, elemPath, key.Interface(), sch); err != nil { + return err + } + if err := validateValue(valueType, elemPath, elem, sch); err != nil { + return err + } + } + typeMatches = true + case *schema.Ref: + decl, ok := sch.Resolve(fieldType).Get() + if !ok { + return fmt.Errorf("unknown ref %v", fieldType) + } + + switch d := decl.(type) { + case *schema.Data: + if valueMap, ok := value.(map[string]any); ok { + if err := validateRequestMap(fieldType, path, valueMap, sch); err != nil { + return err + } + typeMatches = true + } + case *schema.TypeAlias: + return validateValue(d.Type, path, value, sch) + case *schema.Enum: + var inputName any + inputName = value + for _, v := range d.Variants { + switch t := v.Value.(type) { + case *schema.StringValue: + if valueStr, ok := value.(string); ok { + if t.Value == valueStr { + typeMatches = true + break + } + } + case *schema.IntValue: + if valueInt, ok := value.(int); ok { + if t.Value == valueInt { + typeMatches = true + break + } + } + case *schema.TypeValue: + if reqVariant, ok := value.(map[string]any); ok { + vName, ok := reqVariant["name"] + if !ok { + return fmt.Errorf(`missing name field in enum type %q: expected structure is `+ + "{\"name\": \"\", \"value\": }", value) + } + vNameStr, ok := vName.(string) + if !ok { + return fmt.Errorf(`invalid type for enum %q; name field must be a string, was %T`, + fieldType, vName) + } + inputName = fmt.Sprintf("%q", vNameStr) + + vValue, ok := reqVariant["value"] + if !ok { + return fmt.Errorf(`missing value field in enum type %q: expected structure is `+ + "{\"name\": \"\", \"value\": }", value) + } + + if v.Name == vNameStr { + return validateValue(t.Value, path, vValue, sch) + } + } else { + return fmt.Errorf(`malformed enum type %s: expected structure is `+ + "{\"name\": \"\", \"value\": }", path) + } + } + } + if !typeMatches { + return fmt.Errorf("%s is not a valid variant of enum %s", inputName, fieldType) + } + + case *schema.Config, *schema.Database, *schema.Secret, *schema.Verb, *schema.FSM, *schema.Topic, *schema.Subscription: + + } + + case *schema.Bytes: + _, typeMatches = value.([]byte) + if bodyStr, ok := value.(string); ok { + _, err := base64.StdEncoding.DecodeString(bodyStr) + if err != nil { + return fmt.Errorf("%s is not a valid base64 string", path) + } + typeMatches = true + } + + case *schema.Optional: + if value == nil { + typeMatches = true + } else { + return validateValue(fieldType.Type, path, value, sch) + } + } + + if !typeMatches { + return fmt.Errorf("%s has wrong type, expected %s found %T", path, fieldType, value) + } + return nil +} diff --git a/backend/controller/ingress/ingress_test.go b/backend/controller/ingress/ingress_test.go index a448580fbc..1d502f2645 100644 --- a/backend/controller/ingress/ingress_test.go +++ b/backend/controller/ingress/ingress_test.go @@ -143,7 +143,7 @@ func TestValidation(t *testing.T) { sch, err := schema.ParseString("", test.schema) assert.NoError(t, err) - err = schema.ValidateRequestMap(&schema.Ref{Module: "test", Name: "Test"}, nil, test.request, sch) + err = validateRequestMap(&schema.Ref{Module: "test", Name: "Test"}, nil, test.request, sch) if test.err != "" { assert.EqualError(t, err, test.err) } else { @@ -407,7 +407,7 @@ func TestEnumValidation(t *testing.T) { } for _, test := range tests { - err := schema.ValidateJSONValue(test.validateRoot, []string{test.validateRoot.String()}, test.req, sch) + err := validateValue(test.validateRoot, []string{test.validateRoot.String()}, test.req, sch) if test.err == "" { assert.NoError(t, err) } else { diff --git a/backend/controller/ingress/request.go b/backend/controller/ingress/request.go index d9bb18a2b4..4c8ed01e38 100644 --- a/backend/controller/ingress/request.go +++ b/backend/controller/ingress/request.go @@ -2,6 +2,7 @@ package ingress import ( "encoding/json" + "errors" "fmt" "io" "net/http" @@ -83,7 +84,7 @@ func BuildRequestBody(route *dal.IngressRoute, r *http.Request, sch *schema.Sche return nil, err } - err = schema.ValidateRequestMap(request, []string{request.String()}, requestMap, sch) + err = validateRequestMap(request, []string{request.String()}, requestMap, sch) if err != nil { return nil, err } @@ -234,6 +235,45 @@ func buildRequestMap(route *dal.IngressRoute, r *http.Request, ref *schema.Ref, return requestMap, nil } +func validateRequestMap(ref *schema.Ref, path path, request map[string]any, sch *schema.Schema) error { + data, err := sch.ResolveMonomorphised(ref) + if err != nil { + return err + } + + var errs []error + for _, field := range data.Fields { + fieldPath := append(path, "."+field.Name) //nolint:gocritic + + value, haveValue := request[field.Name] + if !haveValue && !allowMissingField(field) { + errs = append(errs, fmt.Errorf("%s is required", fieldPath)) + continue + } + + if haveValue { + err := validateValue(field.Type, fieldPath, value, sch) + if err != nil { + errs = append(errs, err) + } + } + + } + + return errors.Join(errs...) +} + +// Fields of these types can be omitted from the JSON representation. +func allowMissingField(field *schema.Field) bool { + switch field.Type.(type) { + case *schema.Optional, *schema.Any, *schema.Array, *schema.Map, *schema.Bytes, *schema.Unit: + return true + + case *schema.Bool, *schema.Ref, *schema.Float, *schema.Int, *schema.String, *schema.Time: + } + return false +} + func parseQueryParams(values url.Values, data *schema.Data) (map[string]any, error) { if jsonStr, ok := values["@json"]; ok { if len(values) > 1 { diff --git a/backend/schema/jsonvalidate.go b/backend/schema/jsonvalidate.go deleted file mode 100644 index 3e5336b8c2..0000000000 --- a/backend/schema/jsonvalidate.go +++ /dev/null @@ -1,251 +0,0 @@ -package schema - -import ( - "encoding/base64" - "errors" - "fmt" - "reflect" - "strconv" - "strings" - "time" -) - -type path []string - -func (p path) String() string { - return strings.TrimLeft(strings.Join(p, ""), ".") -} - -// ValidateJSONValue validates a given JSON value against the provided schema. -func ValidateJSONValue(fieldType Type, path path, value any, sch *Schema) error { //nolint:maintidx - var typeMatches bool - switch fieldType := fieldType.(type) { - case *Any: - typeMatches = true - - case *Unit: - // TODO: Use type assertions consistently in this function rather than reflection. - rv := reflect.ValueOf(value) - if rv.Kind() != reflect.Map || rv.Len() != 0 { - return fmt.Errorf("%s must be an empty map", path) - } - return nil - - case *Time: - str, ok := value.(string) - if !ok { - return fmt.Errorf("time %s must be an RFC3339 formatted string", path) - } - _, err := time.Parse(time.RFC3339Nano, str) - if err != nil { - return fmt.Errorf("time %s must be an RFC3339 formatted string: %w", path, err) - } - return nil - - case *Int: - switch value := value.(type) { - case int64, float64: - typeMatches = true - case string: - fmt.Printf("found string %s\n", value) - if _, err := strconv.ParseInt(value, 10, 64); err == nil { - typeMatches = true - } - } - - case *Float: - switch value := value.(type) { - case float64: - typeMatches = true - case string: - if _, err := strconv.ParseFloat(value, 64); err == nil { - typeMatches = true - } - } - - case *String: - _, typeMatches = value.(string) - - case *Bool: - switch value := value.(type) { - case bool: - typeMatches = true - case string: - if _, err := strconv.ParseBool(value); err == nil { - typeMatches = true - } - } - - case *Array: - rv := reflect.ValueOf(value) - if rv.Kind() != reflect.Slice { - return fmt.Errorf("%s is not a slice", path) - } - elementType := fieldType.Element - for i := range rv.Len() { - elemPath := append(path, fmt.Sprintf("[%d]", i)) //nolint:gocritic - elem := rv.Index(i).Interface() - if err := ValidateJSONValue(elementType, elemPath, elem, sch); err != nil { - return err - } - } - typeMatches = true - - case *Map: - rv := reflect.ValueOf(value) - if rv.Kind() != reflect.Map { - return fmt.Errorf("%s is not a map", path) - } - keyType := fieldType.Key - valueType := fieldType.Value - for _, key := range rv.MapKeys() { - elemPath := append(path, fmt.Sprintf("[%q]", key)) //nolint:gocritic - elem := rv.MapIndex(key).Interface() - if err := ValidateJSONValue(keyType, elemPath, key.Interface(), sch); err != nil { - return err - } - if err := ValidateJSONValue(valueType, elemPath, elem, sch); err != nil { - return err - } - } - typeMatches = true - case *Ref: - decl, ok := sch.Resolve(fieldType).Get() - if !ok { - return fmt.Errorf("unknown ref %v", fieldType) - } - - switch d := decl.(type) { - case *Data: - if valueMap, ok := value.(map[string]any); ok { - if err := ValidateRequestMap(fieldType, path, valueMap, sch); err != nil { - return err - } - typeMatches = true - } - case *TypeAlias: - return ValidateJSONValue(d.Type, path, value, sch) - case *Enum: - var inputName any - inputName = value - for _, v := range d.Variants { - switch t := v.Value.(type) { - case *StringValue: - if valueStr, ok := value.(string); ok { - if t.Value == valueStr { - typeMatches = true - break - } - } - case *IntValue: - switch value := value.(type) { - case int, int64: - if t.Value == value { - typeMatches = true - break - } - case float64: - if float64(t.Value) == value { - typeMatches = true - break - } - } - case *TypeValue: - if reqVariant, ok := value.(map[string]any); ok { - vName, ok := reqVariant["name"] - if !ok { - return fmt.Errorf(`missing name field in enum type %q: expected structure is `+ - "{\"name\": \"\", \"value\": }", value) - } - vNameStr, ok := vName.(string) - if !ok { - return fmt.Errorf(`invalid type for enum %q; name field must be a string, was %T`, - fieldType, vName) - } - inputName = fmt.Sprintf("%q", vNameStr) - - vValue, ok := reqVariant["value"] - if !ok { - return fmt.Errorf(`missing value field in enum type %q: expected structure is `+ - "{\"name\": \"\", \"value\": }", value) - } - - if v.Name == vNameStr { - return ValidateJSONValue(t.Value, path, vValue, sch) - } - } else { - return fmt.Errorf(`malformed enum type %s: expected structure is `+ - "{\"name\": \"\", \"value\": }", path) - } - } - } - if !typeMatches { - return fmt.Errorf("%s is not a valid variant of enum %s", inputName, fieldType) - } - - case *Config, *Database, *Secret, *Verb, *FSM, *Topic, *Subscription: - - } - - case *Bytes: - _, typeMatches = value.([]byte) - if bodyStr, ok := value.(string); ok { - _, err := base64.StdEncoding.DecodeString(bodyStr) - if err != nil { - return fmt.Errorf("%s is not a valid base64 string", path) - } - typeMatches = true - } - - case *Optional: - if value == nil { - typeMatches = true - } else { - return ValidateJSONValue(fieldType.Type, path, value, sch) - } - } - - if !typeMatches { - return fmt.Errorf("%s has wrong type, expected %s found %T", path, fieldType, value) - } - return nil -} - -func ValidateRequestMap(ref *Ref, path path, request map[string]any, sch *Schema) error { - data, err := sch.ResolveMonomorphised(ref) - if err != nil { - return err - } - - var errs []error - for _, field := range data.Fields { - fieldPath := append(path, "."+field.Name) //nolint:gocritic - - value, haveValue := request[field.Name] - if !haveValue && !allowMissingField(field) { - errs = append(errs, fmt.Errorf("%s is required", fieldPath)) - continue - } - - if haveValue { - err := ValidateJSONValue(field.Type, fieldPath, value, sch) - if err != nil { - errs = append(errs, err) - } - } - - } - - return errors.Join(errs...) -} - -// Fields of these types can be omitted from the JSON representation. -func allowMissingField(field *Field) bool { - switch field.Type.(type) { - case *Optional, *Any, *Array, *Map, *Bytes, *Unit: - return true - - case *Bool, *Ref, *Float, *Int, *String, *Time: - } - return false -} From a6ce60701f9da1c991264725d5e671e9f9295980 Mon Sep 17 00:00:00 2001 From: Alex Szlavik Date: Thu, 4 Jul 2024 20:57:12 -0400 Subject: [PATCH 17/23] feat: Add ca-certificates package to ftl-runner image (#1980) Runner applications egress to internet applications today, requiring basic CA bundles to be present. --- Dockerfile.runner | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.runner b/Dockerfile.runner index 5e3a12bc07..83317006e9 100644 --- a/Dockerfile.runner +++ b/Dockerfile.runner @@ -34,6 +34,8 @@ RUN just build ftl # Finally create the runtime image. FROM ubuntu:24.04 +RUN apt-get update +RUN apt-get install -y ca-certificates WORKDIR /root/ From c38eb92b5bb033ed64e28cbcba587172e147dca6 Mon Sep 17 00:00:00 2001 From: gak Date: Fri, 5 Jul 2024 11:14:03 +1000 Subject: [PATCH 18/23] fix: increase timeout for controller, log timings (#1981) Maybe #1942 * Use `--startup-timeout` flag for controller timeout when in foreground --- cmd/ftl/cmd_serve.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go index 05573c14db..8a418a222d 100644 --- a/cmd/ftl/cmd_serve.go +++ b/cmd/ftl/cmd_serve.go @@ -142,9 +142,11 @@ func (s *serveCmd) run(ctx context.Context, projConfig projectconfig.Config, ini } // Wait for controller to start, then run startup commands. - if err := waitForControllerOnline(ctx, time.Second*10, client); err != nil { + start := time.Now() + if err := waitForControllerOnline(ctx, s.StartupTimeout, client); err != nil { return fmt.Errorf("controller failed to start: %w", err) } + logger.Infof("Controller started in %s", time.Since(start)) if len(projConfig.Commands.Startup) > 0 { for _, cmd := range projConfig.Commands.Startup { @@ -344,6 +346,7 @@ func (s *serveCmd) setupDB(ctx context.Context) (string, error) { // waitForControllerOnline polls the controller service until it is online. func waitForControllerOnline(ctx context.Context, startupTimeout time.Duration, client ftlv1connect.ControllerServiceClient) error { logger := log.FromContext(ctx) + logger.Debugf("Waiting %s for controller to be online", startupTimeout) ctx, cancel := context.WithTimeout(ctx, startupTimeout) defer cancel() From a183b6fec0a0351b47a198d32b5dade8840af546 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 5 Jul 2024 10:49:17 -0700 Subject: [PATCH 19/23] fix: panic in build engine when module building fails (#1985) Fixes #1984 --- buildengine/build.go | 5 +++++ buildengine/engine.go | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/buildengine/build.go b/buildengine/build.go index 2b4fdd7ac9..3a50e0522e 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -41,6 +41,9 @@ func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, } logger.Infof("Building module") + + startTime := time.Now() + switch module.Config.Language { case "go": err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction) @@ -70,6 +73,8 @@ func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, return errors.Join(errs...) } + logger.Infof("Module built (%.2fs)", time.Since(startTime).Seconds()) + return nil } diff --git a/buildengine/engine.go b/buildengine/engine.go index 806acc2d69..dbc109e159 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -729,7 +729,10 @@ func (e *Engine) gatherSchemas( latestModule = moduleMeta{module: module} } for _, dep := range latestModule.module.Dependencies { - out[dep] = moduleSchemas[dep] + if moduleSchemas[dep] != nil { + out[dep] = moduleSchemas[dep] + } + if dep != "builtin" { depModule, ok := e.moduleMetas.Load(dep) // TODO: should we be gathering schemas from dependencies without a module? From 06b763e00821d8a3456ff08d033180ad3316110a Mon Sep 17 00:00:00 2001 From: gak Date: Mon, 8 Jul 2024 08:36:04 +1000 Subject: [PATCH 20/23] fix: demote already closed stream (#1992) Fixes #1956 --- internal/rpc/rpc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/rpc.go b/internal/rpc/rpc.go index 33f8a9bbda..a96a3ec77d 100644 --- a/internal/rpc/rpc.go +++ b/internal/rpc/rpc.go @@ -177,7 +177,7 @@ func RetryStreamingClientStream[Req, Resp any]( // We've hit an error. _, closeErr := stream.CloseAndReceive() if closeErr != nil { - logger.Logf(logLevel, "Failed to close stream: %s", closeErr) + logger.Logf(log.Debug, "Failed to close stream: %s", closeErr) } errored = true From 1c296f54223ff62eb6f7f0b54ae3341740638f86 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Mon, 8 Jul 2024 13:02:02 +1000 Subject: [PATCH 21/23] fix: do not verify obfuscated contents of 1password (#1999) We now obfuscate secrets which caused errors when retrieving values from 1Password because the provider was making sure the value was json --- common/configuration/1password_provider.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/common/configuration/1password_provider.go b/common/configuration/1password_provider.go index 09226ad574..06d65c800f 100644 --- a/common/configuration/1password_provider.go +++ b/common/configuration/1password_provider.go @@ -45,13 +45,6 @@ func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([ return nil, fmt.Errorf("password field not found in item %q", ref) } - // Just to verify that it is JSON encoded. - var decoded interface{} - err = json.Unmarshal(secret, &decoded) - if err != nil { - return nil, fmt.Errorf("secret is not JSON encoded: %w", err) - } - return secret, nil } From e6d84999c5e115a705081eb0d8612a8748859c26 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Mon, 8 Jul 2024 13:24:53 +1000 Subject: [PATCH 22/23] feat: import/export of config and secrets (#1982) closes #1948 **Usage:** **Exporting** Prints json of all secrets to stdout: `ftl secret export` Exports all inline secrets to a json file `ftl secret export --inline > exported.json` **Importing** Imports json from a file: `ftl secret import --inline exported.json` **Migrating** Migrates inline secrets to 1Password: `ftl secret export --inline | ftl secret import --op --opvault ` **JSON Format** ``` { ".": , ... } ``` --- backend/controller/admin/admin.go | 8 +- backend/protos/xyz/block/ftl/v1/ftl.pb.go | 812 +++++++++--------- backend/protos/xyz/block/ftl/v1/ftl.proto | 2 + cmd/ftl/cmd_config.go | 92 +- cmd/ftl/cmd_secret.go | 92 +- cmd/ftl/integration_test.go | 52 ++ cmd/ftl/testdata/import.json | 5 + common/configuration/manager.go | 7 +- .../src/protos/xyz/block/ftl/v1/ftl_pb.ts | 12 + .../compile/compile_integration_test.go | 10 +- integration/actions.go | 19 +- 11 files changed, 702 insertions(+), 409 deletions(-) create mode 100644 cmd/ftl/testdata/import.json diff --git a/backend/controller/admin/admin.go b/backend/controller/admin/admin.go index 3ca0e6ae75..203e5e91ce 100644 --- a/backend/controller/admin/admin.go +++ b/backend/controller/admin/admin.go @@ -40,7 +40,7 @@ func (s *AdminService) ConfigList(ctx context.Context, req *connect.Request[ftlv configs := []*ftlv1.ListConfigResponse_Config{} for _, config := range listing { module, ok := config.Module.Get() - if *req.Msg.Module != "" && module != *req.Msg.Module { + if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module { continue } @@ -127,8 +127,12 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl } secrets := []*ftlv1.ListSecretsResponse_Secret{} for _, secret := range listing { + if req.Msg.Provider != nil && cf.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) { + // Skip secrets that don't match the provider in the request + continue + } module, ok := secret.Module.Get() - if *req.Msg.Module != "" && module != *req.Msg.Module { + if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module { continue } ref := secret.Name diff --git a/backend/protos/xyz/block/ftl/v1/ftl.pb.go b/backend/protos/xyz/block/ftl/v1/ftl.pb.go index 55501a6d2f..779d6a1776 100644 --- a/backend/protos/xyz/block/ftl/v1/ftl.pb.go +++ b/backend/protos/xyz/block/ftl/v1/ftl.pb.go @@ -2704,8 +2704,9 @@ type ListConfigRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Module *string `protobuf:"bytes,1,opt,name=module,proto3,oneof" json:"module,omitempty"` - IncludeValues *bool `protobuf:"varint,2,opt,name=include_values,json=includeValues,proto3,oneof" json:"include_values,omitempty"` + Module *string `protobuf:"bytes,1,opt,name=module,proto3,oneof" json:"module,omitempty"` + IncludeValues *bool `protobuf:"varint,2,opt,name=include_values,json=includeValues,proto3,oneof" json:"include_values,omitempty"` + Provider *ConfigProvider `protobuf:"varint,3,opt,name=provider,proto3,enum=xyz.block.ftl.v1.ConfigProvider,oneof" json:"provider,omitempty"` } func (x *ListConfigRequest) Reset() { @@ -2754,6 +2755,13 @@ func (x *ListConfigRequest) GetIncludeValues() bool { return false } +func (x *ListConfigRequest) GetProvider() ConfigProvider { + if x != nil && x.Provider != nil { + return *x.Provider + } + return ConfigProvider_CONFIG_INLINE +} + type ListConfigResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3094,8 +3102,9 @@ type ListSecretsRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Module *string `protobuf:"bytes,1,opt,name=module,proto3,oneof" json:"module,omitempty"` - IncludeValues *bool `protobuf:"varint,2,opt,name=include_values,json=includeValues,proto3,oneof" json:"include_values,omitempty"` + Module *string `protobuf:"bytes,1,opt,name=module,proto3,oneof" json:"module,omitempty"` + IncludeValues *bool `protobuf:"varint,2,opt,name=include_values,json=includeValues,proto3,oneof" json:"include_values,omitempty"` + Provider *SecretProvider `protobuf:"varint,3,opt,name=provider,proto3,enum=xyz.block.ftl.v1.SecretProvider,oneof" json:"provider,omitempty"` } func (x *ListSecretsRequest) Reset() { @@ -3144,6 +3153,13 @@ func (x *ListSecretsRequest) GetIncludeValues() bool { return false } +func (x *ListSecretsRequest) GetProvider() SecretProvider { + if x != nil && x.Provider != nil { + return *x.Provider + } + return SecretProvider_SECRET_INLINE +} + type ListSecretsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -4744,44 +4760,37 @@ var file_xyz_block_ftl_v1_ftl_proto_rawDesc = []byte{ 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6d, 0x6f, - 0x64, 0x75, 0x6c, 0x65, 0x22, 0x7a, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x06, 0x6d, 0x6f, 0x64, - 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x6d, 0x6f, 0x64, - 0x75, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, - 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x01, - 0x52, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x88, - 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x42, 0x11, 0x0a, - 0x0f, 0x5f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x22, 0xa4, 0x01, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x1a, 0x47, - 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x66, 0x50, - 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x66, 0x50, 0x61, - 0x74, 0x68, 0x12, 0x19, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x48, 0x00, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, - 0x06, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x41, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x72, - 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x29, 0x0a, 0x11, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x78, - 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x48, 0x00, - 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2d, 0x0a, - 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, - 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, - 0x13, 0x0a, 0x11, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x12, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x43, 0x6f, + 0x64, 0x75, 0x6c, 0x65, 0x22, 0xca, 0x01, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x06, 0x6d, 0x6f, + 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x6d, 0x6f, + 0x64, 0x75, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, + 0x01, 0x52, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, + 0x88, 0x01, 0x01, 0x12, 0x41, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, + 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x48, 0x02, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, + 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x73, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x22, 0xa4, 0x01, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x1a, + 0x47, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x66, + 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x66, 0x50, + 0x61, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x48, 0x00, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x88, 0x01, 0x01, 0x42, 0x08, + 0x0a, 0x06, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x41, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x03, + 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x29, 0x0a, 0x11, 0x47, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, @@ -4789,279 +4798,296 @@ var file_xyz_block_ftl_v1_ftl_proto_rawDesc = []byte{ 0x00, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2d, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x42, 0x0b, 0x0a, - 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x15, 0x0a, 0x13, 0x55, 0x6e, - 0x73, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x7b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x6d, 0x6f, 0x64, 0x75, 0x6c, - 0x65, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x01, 0x52, 0x0d, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x88, 0x01, 0x01, - 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x42, 0x11, 0x0a, 0x0f, 0x5f, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0xa6, - 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x47, - 0x0a, 0x06, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x66, 0x50, - 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x66, 0x50, 0x61, - 0x74, 0x68, 0x12, 0x19, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x48, 0x00, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, - 0x06, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x41, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x72, - 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x29, 0x0a, 0x11, 0x47, 0x65, - 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x53, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x78, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x22, 0x13, 0x0a, 0x11, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x12, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x08, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, + 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, + 0x2d, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x48, 0x00, - 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2d, 0x0a, - 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x42, 0x0b, + 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x15, 0x0a, 0x13, 0x55, + 0x6e, 0x73, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0xcb, 0x01, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x06, 0x6d, 0x6f, 0x64, + 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x6d, 0x6f, 0x64, + 0x75, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x01, + 0x52, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x88, + 0x01, 0x01, 0x12, 0x41, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x48, 0x02, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, + 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x73, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x22, 0xa6, 0x01, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, + 0x1a, 0x47, 0x0a, 0x06, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, + 0x66, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x66, + 0x50, 0x61, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x88, 0x01, 0x01, 0x42, + 0x08, 0x0a, 0x06, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x41, 0x0a, 0x10, 0x47, 0x65, 0x74, + 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, + 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, - 0x13, 0x0a, 0x11, 0x53, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x12, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x53, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x08, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, - 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x48, - 0x00, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2d, - 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x42, 0x0b, 0x0a, - 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x15, 0x0a, 0x13, 0x55, 0x6e, - 0x73, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2a, 0x5c, 0x0a, 0x14, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x43, - 0x68, 0x61, 0x6e, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x45, 0x50, - 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x45, 0x44, 0x10, 0x00, 0x12, - 0x16, 0x0a, 0x12, 0x44, 0x45, 0x50, 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x45, - 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x50, 0x4c, 0x4f, - 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x44, 0x10, 0x02, 0x2a, - 0x59, 0x0a, 0x0b, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0f, - 0x0a, 0x0b, 0x52, 0x55, 0x4e, 0x4e, 0x45, 0x52, 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x00, 0x12, - 0x13, 0x0a, 0x0f, 0x52, 0x55, 0x4e, 0x4e, 0x45, 0x52, 0x5f, 0x52, 0x45, 0x53, 0x45, 0x52, 0x56, - 0x45, 0x44, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x55, 0x4e, 0x4e, 0x45, 0x52, 0x5f, 0x41, - 0x53, 0x53, 0x49, 0x47, 0x4e, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x55, 0x4e, - 0x4e, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x41, 0x44, 0x10, 0x03, 0x2a, 0x44, 0x0a, 0x0e, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x11, 0x0a, 0x0d, - 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x49, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x00, 0x12, - 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x45, 0x4e, 0x56, 0x41, 0x52, 0x10, - 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x44, 0x42, 0x10, 0x02, - 0x2a, 0x69, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, 0x49, 0x4e, 0x4c, - 0x49, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, - 0x45, 0x4e, 0x56, 0x41, 0x52, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x43, 0x52, 0x45, - 0x54, 0x5f, 0x4b, 0x45, 0x59, 0x43, 0x48, 0x41, 0x49, 0x4e, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, - 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, 0x4f, 0x50, 0x10, 0x03, 0x12, 0x0e, 0x0a, 0x0a, 0x53, - 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, 0x41, 0x53, 0x4d, 0x10, 0x04, 0x32, 0xa8, 0x04, 0x0a, 0x0b, - 0x56, 0x65, 0x72, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x04, 0x50, - 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, - 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, - 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x12, 0x65, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x4d, 0x6f, - 0x64, 0x75, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x26, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4d, - 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, - 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x61, - 0x0a, 0x0c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x25, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x29, 0x0a, 0x11, + 0x47, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xa7, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x53, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, 0x08, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, - 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, - 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, - 0x01, 0x12, 0x5d, 0x0a, 0x0c, 0x53, 0x65, 0x6e, 0x64, 0x46, 0x53, 0x4d, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x12, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x46, 0x53, 0x4d, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x6e, 0x64, - 0x46, 0x53, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x5d, 0x0a, 0x0c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x31, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, + 0x2d, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x78, + 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x22, 0x13, 0x0a, 0x11, 0x53, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x12, 0x55, 0x6e, 0x73, 0x65, 0x74, + 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x41, 0x0a, + 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x48, 0x00, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, + 0x12, 0x2d, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x42, + 0x0b, 0x0a, 0x09, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x15, 0x0a, 0x13, + 0x55, 0x6e, 0x73, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2a, 0x5c, 0x0a, 0x14, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, + 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x44, + 0x45, 0x50, 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x50, 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x5f, + 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x45, 0x50, + 0x4c, 0x4f, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x44, 0x10, + 0x02, 0x2a, 0x59, 0x0a, 0x0b, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x55, 0x4e, 0x4e, 0x45, 0x52, 0x5f, 0x49, 0x44, 0x4c, 0x45, 0x10, + 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x55, 0x4e, 0x4e, 0x45, 0x52, 0x5f, 0x52, 0x45, 0x53, 0x45, + 0x52, 0x56, 0x45, 0x44, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x52, 0x55, 0x4e, 0x4e, 0x45, 0x52, + 0x5f, 0x41, 0x53, 0x53, 0x49, 0x47, 0x4e, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x52, + 0x55, 0x4e, 0x4e, 0x45, 0x52, 0x5f, 0x44, 0x45, 0x41, 0x44, 0x10, 0x03, 0x2a, 0x44, 0x0a, 0x0e, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x11, + 0x0a, 0x0d, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x49, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, + 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x45, 0x4e, 0x56, 0x41, + 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x5f, 0x44, 0x42, + 0x10, 0x02, 0x2a, 0x69, 0x0a, 0x0e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, 0x49, + 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x45, 0x43, 0x52, 0x45, + 0x54, 0x5f, 0x45, 0x4e, 0x56, 0x41, 0x52, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x43, + 0x52, 0x45, 0x54, 0x5f, 0x4b, 0x45, 0x59, 0x43, 0x48, 0x41, 0x49, 0x4e, 0x10, 0x02, 0x12, 0x0d, + 0x0a, 0x09, 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, 0x4f, 0x50, 0x10, 0x03, 0x12, 0x0e, 0x0a, + 0x0a, 0x53, 0x45, 0x43, 0x52, 0x45, 0x54, 0x5f, 0x41, 0x53, 0x4d, 0x10, 0x04, 0x32, 0xa8, 0x04, + 0x0a, 0x0b, 0x56, 0x65, 0x72, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, + 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x12, 0x65, 0x0a, 0x10, 0x47, 0x65, 0x74, + 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x26, 0x2e, + 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, + 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, + 0x12, 0x61, 0x0a, 0x0c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, - 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, - 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x45, 0x0a, 0x04, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xf6, 0x0a, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x04, - 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, - 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x12, 0x5a, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x63, - 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, - 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, + 0x72, 0x65, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, + 0x01, 0x30, 0x01, 0x12, 0x5d, 0x0a, 0x0c, 0x53, 0x65, 0x6e, 0x64, 0x46, 0x53, 0x4d, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x12, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, + 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x46, 0x53, 0x4d, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, 0x7a, + 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, + 0x6e, 0x64, 0x46, 0x53, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x0c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x12, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, + 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, + 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x45, 0x0a, 0x04, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xf6, 0x0a, 0x0a, 0x11, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, + 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, + 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x12, 0x5a, 0x0a, 0x0b, 0x50, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, + 0x63, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x1f, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, + 0x63, 0x74, 0x44, 0x69, 0x66, 0x66, 0x73, 0x12, 0x29, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, + 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x44, 0x69, 0x66, 0x66, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, + 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, + 0x74, 0x44, 0x69, 0x66, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, + 0x0a, 0x0e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, + 0x12, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, + 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, + 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x6c, + 0x6f, 0x61, 0x64, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x65, 0x70, + 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x29, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, + 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, + 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, + 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x7d, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, + 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x2f, 0x2e, 0x78, 0x79, 0x7a, + 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, + 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x78, 0x79, + 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, + 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x74, 0x65, + 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, + 0x65, 0x0a, 0x0e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, + 0x72, 0x12, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, + 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x78, 0x79, 0x7a, + 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x5d, 0x0a, 0x0c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x12, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, + 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, + 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, + 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x12, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, + 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x69, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, - 0x44, 0x69, 0x66, 0x66, 0x73, 0x12, 0x29, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x65, - 0x66, 0x61, 0x63, 0x74, 0x44, 0x69, 0x66, 0x66, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2a, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x44, - 0x69, 0x66, 0x66, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0e, - 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x12, 0x27, + 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, + 0x2d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, + 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, - 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x69, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, - 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x29, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, - 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2a, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, - 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, - 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x0d, - 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x26, 0x2e, + 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, + 0x12, 0x54, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, - 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7d, - 0x0a, 0x16, 0x47, 0x65, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x41, - 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x2f, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, - 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, 0x63, - 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x78, 0x79, 0x7a, 0x2e, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, - 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x74, 0x65, 0x66, 0x61, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x65, 0x0a, - 0x0e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x12, - 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, - 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x28, 0x01, 0x12, 0x5d, 0x0a, 0x0c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x12, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x12, 0x26, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x44, - 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x0a, 0x50, 0x75, 0x6c, 0x6c, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x12, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x53, 0x63, 0x68, 0x65, + 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x6c, + 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, + 0x01, 0x32, 0xd2, 0x02, 0x0a, 0x0d, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x78, 0x79, + 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, + 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, + 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x12, + 0x4e, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x12, 0x20, 0x2e, 0x78, 0x79, 0x7a, + 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, 0x0a, 0x14, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, - 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x2d, 0x2e, + 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x4b, 0x0a, 0x06, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x12, 0x1f, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x70, + 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x78, 0x79, 0x7a, + 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x09, + 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, + 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, - 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x78, - 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, - 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x54, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x22, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, - 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, 0x0a, 0x0a, 0x50, 0x75, 0x6c, 0x6c, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x12, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, - 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x6c, 0x6c, 0x53, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x32, - 0xd2, 0x02, 0x0a, 0x0d, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x4a, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, - 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x12, 0x4e, 0x0a, - 0x07, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x12, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x78, 0x79, 0x7a, - 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, - 0x06, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x12, 0x1f, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, - 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x70, 0x6c, - 0x6f, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x54, 0x65, - 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, - 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x32, 0x9f, 0x06, 0x0a, 0x0c, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x1d, 0x2e, + 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0x9f, 0x06, 0x0a, 0x0c, 0x41, 0x64, 0x6d, 0x69, 0x6e, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, + 0x1d, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, + 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, + 0x31, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, + 0x90, 0x02, 0x01, 0x12, 0x57, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, + 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x47, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x78, - 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, - 0x01, 0x12, 0x57, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x12, - 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, - 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x47, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x54, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x74, 0x12, + 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, + 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x73, 0x65, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x2e, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x4c, + 0x69, 0x73, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, + 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x54, 0x0a, 0x09, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x47, 0x65, 0x74, 0x12, 0x22, 0x2e, + 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x55, 0x6e, 0x73, 0x65, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x78, 0x79, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, + 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, + 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, - 0x6e, 0x73, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x4c, 0x69, 0x73, - 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, - 0x0a, 0x09, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x47, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x78, 0x79, - 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, - 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x53, 0x65, - 0x74, 0x12, 0x22, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x53, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x12, 0x24, 0x2e, 0x78, 0x79, 0x7a, 0x2e, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x73, - 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, 0x6c, 0x2e, - 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x50, 0x01, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x54, 0x42, 0x44, 0x35, 0x34, 0x35, 0x36, 0x36, 0x39, - 0x37, 0x35, 0x2f, 0x66, 0x74, 0x6c, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x78, 0x79, 0x7a, 0x2f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2f, - 0x66, 0x74, 0x6c, 0x2f, 0x76, 0x31, 0x3b, 0x66, 0x74, 0x6c, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x73, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x25, 0x2e, 0x78, 0x79, 0x7a, 0x2e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x66, 0x74, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x6e, 0x73, 0x65, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x44, 0x50, 0x01, 0x5a, 0x40, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x54, 0x42, 0x44, 0x35, 0x34, 0x35, 0x36, + 0x36, 0x39, 0x37, 0x35, 0x2f, 0x66, 0x74, 0x6c, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x78, 0x79, 0x7a, 0x2f, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x2f, 0x66, 0x74, 0x6c, 0x2f, 0x76, 0x31, 0x3b, 0x66, 0x74, 0x6c, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -5203,98 +5229,100 @@ var file_xyz_block_ftl_v1_ftl_proto_depIdxs = []int32{ 77, // 29: xyz.block.ftl.v1.StatusResponse.ingress_routes:type_name -> xyz.block.ftl.v1.StatusResponse.IngressRoute 78, // 30: xyz.block.ftl.v1.StatusResponse.routes:type_name -> xyz.block.ftl.v1.StatusResponse.Route 80, // 31: xyz.block.ftl.v1.ProcessListResponse.processes:type_name -> xyz.block.ftl.v1.ProcessListResponse.Process - 81, // 32: xyz.block.ftl.v1.ListConfigResponse.configs:type_name -> xyz.block.ftl.v1.ListConfigResponse.Config - 50, // 33: xyz.block.ftl.v1.GetConfigRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef - 2, // 34: xyz.block.ftl.v1.SetConfigRequest.provider:type_name -> xyz.block.ftl.v1.ConfigProvider - 50, // 35: xyz.block.ftl.v1.SetConfigRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef - 2, // 36: xyz.block.ftl.v1.UnsetConfigRequest.provider:type_name -> xyz.block.ftl.v1.ConfigProvider - 50, // 37: xyz.block.ftl.v1.UnsetConfigRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef - 82, // 38: xyz.block.ftl.v1.ListSecretsResponse.secrets:type_name -> xyz.block.ftl.v1.ListSecretsResponse.Secret - 50, // 39: xyz.block.ftl.v1.GetSecretRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef - 3, // 40: xyz.block.ftl.v1.SetSecretRequest.provider:type_name -> xyz.block.ftl.v1.SecretProvider - 50, // 41: xyz.block.ftl.v1.SetSecretRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef - 3, // 42: xyz.block.ftl.v1.UnsetSecretRequest.provider:type_name -> xyz.block.ftl.v1.SecretProvider - 50, // 43: xyz.block.ftl.v1.UnsetSecretRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef - 4, // 44: xyz.block.ftl.v1.ModuleContextResponse.DSN.type:type_name -> xyz.block.ftl.v1.ModuleContextResponse.DBType - 1, // 45: xyz.block.ftl.v1.StatusResponse.Runner.state:type_name -> xyz.block.ftl.v1.RunnerState - 88, // 46: xyz.block.ftl.v1.StatusResponse.Runner.labels:type_name -> google.protobuf.Struct - 88, // 47: xyz.block.ftl.v1.StatusResponse.Deployment.labels:type_name -> google.protobuf.Struct - 87, // 48: xyz.block.ftl.v1.StatusResponse.Deployment.schema:type_name -> xyz.block.ftl.v1.schema.Module - 83, // 49: xyz.block.ftl.v1.StatusResponse.IngressRoute.verb:type_name -> xyz.block.ftl.v1.schema.Ref - 88, // 50: xyz.block.ftl.v1.ProcessListResponse.ProcessRunner.labels:type_name -> google.protobuf.Struct - 88, // 51: xyz.block.ftl.v1.ProcessListResponse.Process.labels:type_name -> google.protobuf.Struct - 79, // 52: xyz.block.ftl.v1.ProcessListResponse.Process.runner:type_name -> xyz.block.ftl.v1.ProcessListResponse.ProcessRunner - 5, // 53: xyz.block.ftl.v1.VerbService.Ping:input_type -> xyz.block.ftl.v1.PingRequest - 7, // 54: xyz.block.ftl.v1.VerbService.GetModuleContext:input_type -> xyz.block.ftl.v1.ModuleContextRequest - 12, // 55: xyz.block.ftl.v1.VerbService.AcquireLease:input_type -> xyz.block.ftl.v1.AcquireLeaseRequest - 14, // 56: xyz.block.ftl.v1.VerbService.SendFSMEvent:input_type -> xyz.block.ftl.v1.SendFSMEventRequest - 16, // 57: xyz.block.ftl.v1.VerbService.PublishEvent:input_type -> xyz.block.ftl.v1.PublishEventRequest - 10, // 58: xyz.block.ftl.v1.VerbService.Call:input_type -> xyz.block.ftl.v1.CallRequest - 5, // 59: xyz.block.ftl.v1.ControllerService.Ping:input_type -> xyz.block.ftl.v1.PingRequest - 43, // 60: xyz.block.ftl.v1.ControllerService.ProcessList:input_type -> xyz.block.ftl.v1.ProcessListRequest - 41, // 61: xyz.block.ftl.v1.ControllerService.Status:input_type -> xyz.block.ftl.v1.StatusRequest - 22, // 62: xyz.block.ftl.v1.ControllerService.GetArtefactDiffs:input_type -> xyz.block.ftl.v1.GetArtefactDiffsRequest - 24, // 63: xyz.block.ftl.v1.ControllerService.UploadArtefact:input_type -> xyz.block.ftl.v1.UploadArtefactRequest - 27, // 64: xyz.block.ftl.v1.ControllerService.CreateDeployment:input_type -> xyz.block.ftl.v1.CreateDeploymentRequest - 31, // 65: xyz.block.ftl.v1.ControllerService.GetDeployment:input_type -> xyz.block.ftl.v1.GetDeploymentRequest - 29, // 66: xyz.block.ftl.v1.ControllerService.GetDeploymentArtefacts:input_type -> xyz.block.ftl.v1.GetDeploymentArtefactsRequest - 33, // 67: xyz.block.ftl.v1.ControllerService.RegisterRunner:input_type -> xyz.block.ftl.v1.RegisterRunnerRequest - 35, // 68: xyz.block.ftl.v1.ControllerService.UpdateDeploy:input_type -> xyz.block.ftl.v1.UpdateDeployRequest - 37, // 69: xyz.block.ftl.v1.ControllerService.ReplaceDeploy:input_type -> xyz.block.ftl.v1.ReplaceDeployRequest - 39, // 70: xyz.block.ftl.v1.ControllerService.StreamDeploymentLogs:input_type -> xyz.block.ftl.v1.StreamDeploymentLogsRequest - 18, // 71: xyz.block.ftl.v1.ControllerService.GetSchema:input_type -> xyz.block.ftl.v1.GetSchemaRequest - 20, // 72: xyz.block.ftl.v1.ControllerService.PullSchema:input_type -> xyz.block.ftl.v1.PullSchemaRequest - 5, // 73: xyz.block.ftl.v1.RunnerService.Ping:input_type -> xyz.block.ftl.v1.PingRequest - 48, // 74: xyz.block.ftl.v1.RunnerService.Reserve:input_type -> xyz.block.ftl.v1.ReserveRequest - 45, // 75: xyz.block.ftl.v1.RunnerService.Deploy:input_type -> xyz.block.ftl.v1.DeployRequest - 47, // 76: xyz.block.ftl.v1.RunnerService.Terminate:input_type -> xyz.block.ftl.v1.TerminateRequest - 5, // 77: xyz.block.ftl.v1.AdminService.Ping:input_type -> xyz.block.ftl.v1.PingRequest - 51, // 78: xyz.block.ftl.v1.AdminService.ConfigList:input_type -> xyz.block.ftl.v1.ListConfigRequest - 53, // 79: xyz.block.ftl.v1.AdminService.ConfigGet:input_type -> xyz.block.ftl.v1.GetConfigRequest - 55, // 80: xyz.block.ftl.v1.AdminService.ConfigSet:input_type -> xyz.block.ftl.v1.SetConfigRequest - 57, // 81: xyz.block.ftl.v1.AdminService.ConfigUnset:input_type -> xyz.block.ftl.v1.UnsetConfigRequest - 59, // 82: xyz.block.ftl.v1.AdminService.SecretsList:input_type -> xyz.block.ftl.v1.ListSecretsRequest - 61, // 83: xyz.block.ftl.v1.AdminService.SecretGet:input_type -> xyz.block.ftl.v1.GetSecretRequest - 63, // 84: xyz.block.ftl.v1.AdminService.SecretSet:input_type -> xyz.block.ftl.v1.SetSecretRequest - 65, // 85: xyz.block.ftl.v1.AdminService.SecretUnset:input_type -> xyz.block.ftl.v1.UnsetSecretRequest - 6, // 86: xyz.block.ftl.v1.VerbService.Ping:output_type -> xyz.block.ftl.v1.PingResponse - 8, // 87: xyz.block.ftl.v1.VerbService.GetModuleContext:output_type -> xyz.block.ftl.v1.ModuleContextResponse - 13, // 88: xyz.block.ftl.v1.VerbService.AcquireLease:output_type -> xyz.block.ftl.v1.AcquireLeaseResponse - 15, // 89: xyz.block.ftl.v1.VerbService.SendFSMEvent:output_type -> xyz.block.ftl.v1.SendFSMEventResponse - 17, // 90: xyz.block.ftl.v1.VerbService.PublishEvent:output_type -> xyz.block.ftl.v1.PublishEventResponse - 11, // 91: xyz.block.ftl.v1.VerbService.Call:output_type -> xyz.block.ftl.v1.CallResponse - 6, // 92: xyz.block.ftl.v1.ControllerService.Ping:output_type -> xyz.block.ftl.v1.PingResponse - 44, // 93: xyz.block.ftl.v1.ControllerService.ProcessList:output_type -> xyz.block.ftl.v1.ProcessListResponse - 42, // 94: xyz.block.ftl.v1.ControllerService.Status:output_type -> xyz.block.ftl.v1.StatusResponse - 23, // 95: xyz.block.ftl.v1.ControllerService.GetArtefactDiffs:output_type -> xyz.block.ftl.v1.GetArtefactDiffsResponse - 25, // 96: xyz.block.ftl.v1.ControllerService.UploadArtefact:output_type -> xyz.block.ftl.v1.UploadArtefactResponse - 28, // 97: xyz.block.ftl.v1.ControllerService.CreateDeployment:output_type -> xyz.block.ftl.v1.CreateDeploymentResponse - 32, // 98: xyz.block.ftl.v1.ControllerService.GetDeployment:output_type -> xyz.block.ftl.v1.GetDeploymentResponse - 30, // 99: xyz.block.ftl.v1.ControllerService.GetDeploymentArtefacts:output_type -> xyz.block.ftl.v1.GetDeploymentArtefactsResponse - 34, // 100: xyz.block.ftl.v1.ControllerService.RegisterRunner:output_type -> xyz.block.ftl.v1.RegisterRunnerResponse - 36, // 101: xyz.block.ftl.v1.ControllerService.UpdateDeploy:output_type -> xyz.block.ftl.v1.UpdateDeployResponse - 38, // 102: xyz.block.ftl.v1.ControllerService.ReplaceDeploy:output_type -> xyz.block.ftl.v1.ReplaceDeployResponse - 40, // 103: xyz.block.ftl.v1.ControllerService.StreamDeploymentLogs:output_type -> xyz.block.ftl.v1.StreamDeploymentLogsResponse - 19, // 104: xyz.block.ftl.v1.ControllerService.GetSchema:output_type -> xyz.block.ftl.v1.GetSchemaResponse - 21, // 105: xyz.block.ftl.v1.ControllerService.PullSchema:output_type -> xyz.block.ftl.v1.PullSchemaResponse - 6, // 106: xyz.block.ftl.v1.RunnerService.Ping:output_type -> xyz.block.ftl.v1.PingResponse - 49, // 107: xyz.block.ftl.v1.RunnerService.Reserve:output_type -> xyz.block.ftl.v1.ReserveResponse - 46, // 108: xyz.block.ftl.v1.RunnerService.Deploy:output_type -> xyz.block.ftl.v1.DeployResponse - 33, // 109: xyz.block.ftl.v1.RunnerService.Terminate:output_type -> xyz.block.ftl.v1.RegisterRunnerRequest - 6, // 110: xyz.block.ftl.v1.AdminService.Ping:output_type -> xyz.block.ftl.v1.PingResponse - 52, // 111: xyz.block.ftl.v1.AdminService.ConfigList:output_type -> xyz.block.ftl.v1.ListConfigResponse - 54, // 112: xyz.block.ftl.v1.AdminService.ConfigGet:output_type -> xyz.block.ftl.v1.GetConfigResponse - 56, // 113: xyz.block.ftl.v1.AdminService.ConfigSet:output_type -> xyz.block.ftl.v1.SetConfigResponse - 58, // 114: xyz.block.ftl.v1.AdminService.ConfigUnset:output_type -> xyz.block.ftl.v1.UnsetConfigResponse - 60, // 115: xyz.block.ftl.v1.AdminService.SecretsList:output_type -> xyz.block.ftl.v1.ListSecretsResponse - 62, // 116: xyz.block.ftl.v1.AdminService.SecretGet:output_type -> xyz.block.ftl.v1.GetSecretResponse - 64, // 117: xyz.block.ftl.v1.AdminService.SecretSet:output_type -> xyz.block.ftl.v1.SetSecretResponse - 66, // 118: xyz.block.ftl.v1.AdminService.SecretUnset:output_type -> xyz.block.ftl.v1.UnsetSecretResponse - 86, // [86:119] is the sub-list for method output_type - 53, // [53:86] is the sub-list for method input_type - 53, // [53:53] is the sub-list for extension type_name - 53, // [53:53] is the sub-list for extension extendee - 0, // [0:53] is the sub-list for field type_name + 2, // 32: xyz.block.ftl.v1.ListConfigRequest.provider:type_name -> xyz.block.ftl.v1.ConfigProvider + 81, // 33: xyz.block.ftl.v1.ListConfigResponse.configs:type_name -> xyz.block.ftl.v1.ListConfigResponse.Config + 50, // 34: xyz.block.ftl.v1.GetConfigRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef + 2, // 35: xyz.block.ftl.v1.SetConfigRequest.provider:type_name -> xyz.block.ftl.v1.ConfigProvider + 50, // 36: xyz.block.ftl.v1.SetConfigRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef + 2, // 37: xyz.block.ftl.v1.UnsetConfigRequest.provider:type_name -> xyz.block.ftl.v1.ConfigProvider + 50, // 38: xyz.block.ftl.v1.UnsetConfigRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef + 3, // 39: xyz.block.ftl.v1.ListSecretsRequest.provider:type_name -> xyz.block.ftl.v1.SecretProvider + 82, // 40: xyz.block.ftl.v1.ListSecretsResponse.secrets:type_name -> xyz.block.ftl.v1.ListSecretsResponse.Secret + 50, // 41: xyz.block.ftl.v1.GetSecretRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef + 3, // 42: xyz.block.ftl.v1.SetSecretRequest.provider:type_name -> xyz.block.ftl.v1.SecretProvider + 50, // 43: xyz.block.ftl.v1.SetSecretRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef + 3, // 44: xyz.block.ftl.v1.UnsetSecretRequest.provider:type_name -> xyz.block.ftl.v1.SecretProvider + 50, // 45: xyz.block.ftl.v1.UnsetSecretRequest.ref:type_name -> xyz.block.ftl.v1.ConfigRef + 4, // 46: xyz.block.ftl.v1.ModuleContextResponse.DSN.type:type_name -> xyz.block.ftl.v1.ModuleContextResponse.DBType + 1, // 47: xyz.block.ftl.v1.StatusResponse.Runner.state:type_name -> xyz.block.ftl.v1.RunnerState + 88, // 48: xyz.block.ftl.v1.StatusResponse.Runner.labels:type_name -> google.protobuf.Struct + 88, // 49: xyz.block.ftl.v1.StatusResponse.Deployment.labels:type_name -> google.protobuf.Struct + 87, // 50: xyz.block.ftl.v1.StatusResponse.Deployment.schema:type_name -> xyz.block.ftl.v1.schema.Module + 83, // 51: xyz.block.ftl.v1.StatusResponse.IngressRoute.verb:type_name -> xyz.block.ftl.v1.schema.Ref + 88, // 52: xyz.block.ftl.v1.ProcessListResponse.ProcessRunner.labels:type_name -> google.protobuf.Struct + 88, // 53: xyz.block.ftl.v1.ProcessListResponse.Process.labels:type_name -> google.protobuf.Struct + 79, // 54: xyz.block.ftl.v1.ProcessListResponse.Process.runner:type_name -> xyz.block.ftl.v1.ProcessListResponse.ProcessRunner + 5, // 55: xyz.block.ftl.v1.VerbService.Ping:input_type -> xyz.block.ftl.v1.PingRequest + 7, // 56: xyz.block.ftl.v1.VerbService.GetModuleContext:input_type -> xyz.block.ftl.v1.ModuleContextRequest + 12, // 57: xyz.block.ftl.v1.VerbService.AcquireLease:input_type -> xyz.block.ftl.v1.AcquireLeaseRequest + 14, // 58: xyz.block.ftl.v1.VerbService.SendFSMEvent:input_type -> xyz.block.ftl.v1.SendFSMEventRequest + 16, // 59: xyz.block.ftl.v1.VerbService.PublishEvent:input_type -> xyz.block.ftl.v1.PublishEventRequest + 10, // 60: xyz.block.ftl.v1.VerbService.Call:input_type -> xyz.block.ftl.v1.CallRequest + 5, // 61: xyz.block.ftl.v1.ControllerService.Ping:input_type -> xyz.block.ftl.v1.PingRequest + 43, // 62: xyz.block.ftl.v1.ControllerService.ProcessList:input_type -> xyz.block.ftl.v1.ProcessListRequest + 41, // 63: xyz.block.ftl.v1.ControllerService.Status:input_type -> xyz.block.ftl.v1.StatusRequest + 22, // 64: xyz.block.ftl.v1.ControllerService.GetArtefactDiffs:input_type -> xyz.block.ftl.v1.GetArtefactDiffsRequest + 24, // 65: xyz.block.ftl.v1.ControllerService.UploadArtefact:input_type -> xyz.block.ftl.v1.UploadArtefactRequest + 27, // 66: xyz.block.ftl.v1.ControllerService.CreateDeployment:input_type -> xyz.block.ftl.v1.CreateDeploymentRequest + 31, // 67: xyz.block.ftl.v1.ControllerService.GetDeployment:input_type -> xyz.block.ftl.v1.GetDeploymentRequest + 29, // 68: xyz.block.ftl.v1.ControllerService.GetDeploymentArtefacts:input_type -> xyz.block.ftl.v1.GetDeploymentArtefactsRequest + 33, // 69: xyz.block.ftl.v1.ControllerService.RegisterRunner:input_type -> xyz.block.ftl.v1.RegisterRunnerRequest + 35, // 70: xyz.block.ftl.v1.ControllerService.UpdateDeploy:input_type -> xyz.block.ftl.v1.UpdateDeployRequest + 37, // 71: xyz.block.ftl.v1.ControllerService.ReplaceDeploy:input_type -> xyz.block.ftl.v1.ReplaceDeployRequest + 39, // 72: xyz.block.ftl.v1.ControllerService.StreamDeploymentLogs:input_type -> xyz.block.ftl.v1.StreamDeploymentLogsRequest + 18, // 73: xyz.block.ftl.v1.ControllerService.GetSchema:input_type -> xyz.block.ftl.v1.GetSchemaRequest + 20, // 74: xyz.block.ftl.v1.ControllerService.PullSchema:input_type -> xyz.block.ftl.v1.PullSchemaRequest + 5, // 75: xyz.block.ftl.v1.RunnerService.Ping:input_type -> xyz.block.ftl.v1.PingRequest + 48, // 76: xyz.block.ftl.v1.RunnerService.Reserve:input_type -> xyz.block.ftl.v1.ReserveRequest + 45, // 77: xyz.block.ftl.v1.RunnerService.Deploy:input_type -> xyz.block.ftl.v1.DeployRequest + 47, // 78: xyz.block.ftl.v1.RunnerService.Terminate:input_type -> xyz.block.ftl.v1.TerminateRequest + 5, // 79: xyz.block.ftl.v1.AdminService.Ping:input_type -> xyz.block.ftl.v1.PingRequest + 51, // 80: xyz.block.ftl.v1.AdminService.ConfigList:input_type -> xyz.block.ftl.v1.ListConfigRequest + 53, // 81: xyz.block.ftl.v1.AdminService.ConfigGet:input_type -> xyz.block.ftl.v1.GetConfigRequest + 55, // 82: xyz.block.ftl.v1.AdminService.ConfigSet:input_type -> xyz.block.ftl.v1.SetConfigRequest + 57, // 83: xyz.block.ftl.v1.AdminService.ConfigUnset:input_type -> xyz.block.ftl.v1.UnsetConfigRequest + 59, // 84: xyz.block.ftl.v1.AdminService.SecretsList:input_type -> xyz.block.ftl.v1.ListSecretsRequest + 61, // 85: xyz.block.ftl.v1.AdminService.SecretGet:input_type -> xyz.block.ftl.v1.GetSecretRequest + 63, // 86: xyz.block.ftl.v1.AdminService.SecretSet:input_type -> xyz.block.ftl.v1.SetSecretRequest + 65, // 87: xyz.block.ftl.v1.AdminService.SecretUnset:input_type -> xyz.block.ftl.v1.UnsetSecretRequest + 6, // 88: xyz.block.ftl.v1.VerbService.Ping:output_type -> xyz.block.ftl.v1.PingResponse + 8, // 89: xyz.block.ftl.v1.VerbService.GetModuleContext:output_type -> xyz.block.ftl.v1.ModuleContextResponse + 13, // 90: xyz.block.ftl.v1.VerbService.AcquireLease:output_type -> xyz.block.ftl.v1.AcquireLeaseResponse + 15, // 91: xyz.block.ftl.v1.VerbService.SendFSMEvent:output_type -> xyz.block.ftl.v1.SendFSMEventResponse + 17, // 92: xyz.block.ftl.v1.VerbService.PublishEvent:output_type -> xyz.block.ftl.v1.PublishEventResponse + 11, // 93: xyz.block.ftl.v1.VerbService.Call:output_type -> xyz.block.ftl.v1.CallResponse + 6, // 94: xyz.block.ftl.v1.ControllerService.Ping:output_type -> xyz.block.ftl.v1.PingResponse + 44, // 95: xyz.block.ftl.v1.ControllerService.ProcessList:output_type -> xyz.block.ftl.v1.ProcessListResponse + 42, // 96: xyz.block.ftl.v1.ControllerService.Status:output_type -> xyz.block.ftl.v1.StatusResponse + 23, // 97: xyz.block.ftl.v1.ControllerService.GetArtefactDiffs:output_type -> xyz.block.ftl.v1.GetArtefactDiffsResponse + 25, // 98: xyz.block.ftl.v1.ControllerService.UploadArtefact:output_type -> xyz.block.ftl.v1.UploadArtefactResponse + 28, // 99: xyz.block.ftl.v1.ControllerService.CreateDeployment:output_type -> xyz.block.ftl.v1.CreateDeploymentResponse + 32, // 100: xyz.block.ftl.v1.ControllerService.GetDeployment:output_type -> xyz.block.ftl.v1.GetDeploymentResponse + 30, // 101: xyz.block.ftl.v1.ControllerService.GetDeploymentArtefacts:output_type -> xyz.block.ftl.v1.GetDeploymentArtefactsResponse + 34, // 102: xyz.block.ftl.v1.ControllerService.RegisterRunner:output_type -> xyz.block.ftl.v1.RegisterRunnerResponse + 36, // 103: xyz.block.ftl.v1.ControllerService.UpdateDeploy:output_type -> xyz.block.ftl.v1.UpdateDeployResponse + 38, // 104: xyz.block.ftl.v1.ControllerService.ReplaceDeploy:output_type -> xyz.block.ftl.v1.ReplaceDeployResponse + 40, // 105: xyz.block.ftl.v1.ControllerService.StreamDeploymentLogs:output_type -> xyz.block.ftl.v1.StreamDeploymentLogsResponse + 19, // 106: xyz.block.ftl.v1.ControllerService.GetSchema:output_type -> xyz.block.ftl.v1.GetSchemaResponse + 21, // 107: xyz.block.ftl.v1.ControllerService.PullSchema:output_type -> xyz.block.ftl.v1.PullSchemaResponse + 6, // 108: xyz.block.ftl.v1.RunnerService.Ping:output_type -> xyz.block.ftl.v1.PingResponse + 49, // 109: xyz.block.ftl.v1.RunnerService.Reserve:output_type -> xyz.block.ftl.v1.ReserveResponse + 46, // 110: xyz.block.ftl.v1.RunnerService.Deploy:output_type -> xyz.block.ftl.v1.DeployResponse + 33, // 111: xyz.block.ftl.v1.RunnerService.Terminate:output_type -> xyz.block.ftl.v1.RegisterRunnerRequest + 6, // 112: xyz.block.ftl.v1.AdminService.Ping:output_type -> xyz.block.ftl.v1.PingResponse + 52, // 113: xyz.block.ftl.v1.AdminService.ConfigList:output_type -> xyz.block.ftl.v1.ListConfigResponse + 54, // 114: xyz.block.ftl.v1.AdminService.ConfigGet:output_type -> xyz.block.ftl.v1.GetConfigResponse + 56, // 115: xyz.block.ftl.v1.AdminService.ConfigSet:output_type -> xyz.block.ftl.v1.SetConfigResponse + 58, // 116: xyz.block.ftl.v1.AdminService.ConfigUnset:output_type -> xyz.block.ftl.v1.UnsetConfigResponse + 60, // 117: xyz.block.ftl.v1.AdminService.SecretsList:output_type -> xyz.block.ftl.v1.ListSecretsResponse + 62, // 118: xyz.block.ftl.v1.AdminService.SecretGet:output_type -> xyz.block.ftl.v1.GetSecretResponse + 64, // 119: xyz.block.ftl.v1.AdminService.SecretSet:output_type -> xyz.block.ftl.v1.SetSecretResponse + 66, // 120: xyz.block.ftl.v1.AdminService.SecretUnset:output_type -> xyz.block.ftl.v1.UnsetSecretResponse + 88, // [88:121] is the sub-list for method output_type + 55, // [55:88] is the sub-list for method input_type + 55, // [55:55] is the sub-list for extension type_name + 55, // [55:55] is the sub-list for extension extendee + 0, // [0:55] is the sub-list for field type_name } func init() { file_xyz_block_ftl_v1_ftl_proto_init() } diff --git a/backend/protos/xyz/block/ftl/v1/ftl.proto b/backend/protos/xyz/block/ftl/v1/ftl.proto index d593fb9dbd..810d9b768b 100644 --- a/backend/protos/xyz/block/ftl/v1/ftl.proto +++ b/backend/protos/xyz/block/ftl/v1/ftl.proto @@ -414,6 +414,7 @@ enum ConfigProvider { message ListConfigRequest { optional string module = 1; optional bool include_values = 2; + optional ConfigProvider provider = 3; } message ListConfigResponse { message Config { @@ -463,6 +464,7 @@ enum SecretProvider { message ListSecretsRequest { optional string module = 1; optional bool include_values = 2; + optional SecretProvider provider = 3; } message ListSecretsResponse { message Secret { diff --git a/cmd/ftl/cmd_config.go b/cmd/ftl/cmd_config.go index b818adc095..634d6b38ec 100644 --- a/cmd/ftl/cmd_config.go +++ b/cmd/ftl/cmd_config.go @@ -17,10 +17,12 @@ import ( ) type configCmd struct { - List configListCmd `cmd:"" help:"List configuration."` - Get configGetCmd `cmd:"" help:"Get a configuration value."` - Set configSetCmd `cmd:"" help:"Set a configuration value."` - Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."` + List configListCmd `cmd:"" help:"List configuration."` + Get configGetCmd `cmd:"" help:"Get a configuration value."` + Set configSetCmd `cmd:"" help:"Set a configuration value."` + Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."` + Import configImportCmd `cmd:"" help:"Import configuration values."` + Export configExportCmd `cmd:"" help:"Export configuration values."` Envar bool `help:"Print configuration as environment variables." group:"Provider:" xor:"configwriter"` Inline bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"configwriter"` @@ -162,3 +164,85 @@ func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd, adminClient a } return nil } + +type configImportCmd struct { + Input *os.File `arg:"" placeholder:"JSON" help:"JSON to import as configuration values (read from stdin if omitted). Format: {\".\": , ... }" optional:"" default:"-"` +} + +func (s *configImportCmd) Help() string { + return ` +Imports configuration values from a JSON object. +` +} + +func (s *configImportCmd) Run(ctx context.Context, cmd *configCmd, adminClient admin.Client) error { + input, err := io.ReadAll(s.Input) + if err != nil { + return fmt.Errorf("failed to read input: %w", err) + } + var entries map[string]json.RawMessage + err = json.Unmarshal(input, &entries) + if err != nil { + return fmt.Errorf("could not parse JSON: %w", err) + } + for refPath, value := range entries { + ref, err := cf.ParseRef(refPath) + if err != nil { + return fmt.Errorf("could not parse ref %q: %w", refPath, err) + } + bytes, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("could not marshal value for %q: %w", refPath, err) + } + req := &ftlv1.SetConfigRequest{ + Ref: configRefFromRef(ref), + Value: bytes, + } + if provider, ok := cmd.provider().Get(); ok { + req.Provider = &provider + } + _, err = adminClient.ConfigSet(ctx, connect.NewRequest(req)) + if err != nil { + return fmt.Errorf("could not import config for %q: %w", refPath, err) + } + } + return nil +} + +type configExportCmd struct { +} + +func (s *configExportCmd) Help() string { + return ` +Outputs configuration values in a JSON object. A provider can be used to filter which values are included. +` +} + +func (s *configExportCmd) Run(ctx context.Context, cmd *configCmd, adminClient admin.Client) error { + req := &ftlv1.ListConfigRequest{ + IncludeValues: optional.Some(true).Ptr(), + } + if provider, ok := cmd.provider().Get(); ok { + req.Provider = &provider + } + listResponse, err := adminClient.ConfigList(ctx, connect.NewRequest(req)) + if err != nil { + return fmt.Errorf("could not retrieve configs: %w", err) + } + entries := make(map[string]json.RawMessage, 0) + for _, config := range listResponse.Msg.Configs { + var value json.RawMessage + err = json.Unmarshal(config.Value, &value) + if err != nil { + return fmt.Errorf("could not export %q: %w", config.RefPath, err) + } + entries[config.RefPath] = value + } + + output, err := json.Marshal(entries) + if err != nil { + return fmt.Errorf("could not build output: %w", err) + } + fmt.Println(string(output)) + return nil +} diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go index 75a58c4588..1e969031c0 100644 --- a/cmd/ftl/cmd_secret.go +++ b/cmd/ftl/cmd_secret.go @@ -19,10 +19,12 @@ import ( ) type secretCmd struct { - List secretListCmd `cmd:"" help:"List secrets."` - Get secretGetCmd `cmd:"" help:"Get a secret."` - Set secretSetCmd `cmd:"" help:"Set a secret."` - Unset secretUnsetCmd `cmd:"" help:"Unset a secret."` + List secretListCmd `cmd:"" help:"List secrets."` + Get secretGetCmd `cmd:"" help:"Get a secret."` + Set secretSetCmd `cmd:"" help:"Set a secret."` + Unset secretUnsetCmd `cmd:"" help:"Unset a secret."` + Import secretImportCmd `cmd:"" help:"Import secrets."` + Export secretExportCmd `cmd:"" help:"Export secrets."` Envar bool `help:"Write configuration as environment variables." group:"Provider:" xor:"secretwriter"` Inline bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"secretwriter"` @@ -169,3 +171,85 @@ func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd, adminClient a } return nil } + +type secretImportCmd struct { + Input *os.File `arg:"" placeholder:"JSON" help:"JSON to import as secrets (read from stdin if omitted). Format: {\".\": , ... }" optional:"" default:"-"` +} + +func (s *secretImportCmd) Help() string { + return ` +Imports secrets from a JSON object. +` +} + +func (s *secretImportCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error { + input, err := io.ReadAll(s.Input) + if err != nil { + return fmt.Errorf("failed to read input: %w", err) + } + var entries map[string]json.RawMessage + err = json.Unmarshal(input, &entries) + if err != nil { + return fmt.Errorf("could not parse JSON: %w", err) + } + for refPath, value := range entries { + ref, err := cf.ParseRef(refPath) + if err != nil { + return fmt.Errorf("could not parse ref %q: %w", refPath, err) + } + bytes, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("could not marshal value for %q: %w", refPath, err) + } + req := &ftlv1.SetSecretRequest{ + Ref: configRefFromRef(ref), + Value: bytes, + } + if provider, ok := scmd.provider().Get(); ok { + req.Provider = &provider + } + _, err = adminClient.SecretSet(ctx, connect.NewRequest(req)) + if err != nil { + return fmt.Errorf("could not import secret for %q: %w", refPath, err) + } + } + return nil +} + +type secretExportCmd struct { +} + +func (s *secretExportCmd) Help() string { + return ` +Outputs secrets in a JSON object. A provider can be used to filter which secrets are included. +` +} + +func (s *secretExportCmd) Run(ctx context.Context, scmd *secretCmd, adminClient admin.Client) error { + req := &ftlv1.ListSecretsRequest{ + IncludeValues: optional.Some(true).Ptr(), + } + if provider, ok := scmd.provider().Get(); ok { + req.Provider = &provider + } + listResponse, err := adminClient.SecretsList(ctx, connect.NewRequest(req)) + if err != nil { + return fmt.Errorf("could not retrieve secrets: %w", err) + } + entries := make(map[string]json.RawMessage, 0) + for _, secret := range listResponse.Msg.Secrets { + var value json.RawMessage + err = json.Unmarshal(secret.Value, &value) + if err != nil { + return fmt.Errorf("could not export %q: %w", secret.RefPath, err) + } + entries[secret.RefPath] = value + } + + output, err := json.Marshal(entries) + if err != nil { + return fmt.Errorf("could not build output: %w", err) + } + fmt.Println(string(output)) + return nil +} diff --git a/cmd/ftl/integration_test.go b/cmd/ftl/integration_test.go index 68354920dd..3789c89a8f 100644 --- a/cmd/ftl/integration_test.go +++ b/cmd/ftl/integration_test.go @@ -4,6 +4,7 @@ package main import ( "context" + "path/filepath" "testing" "github.com/alecthomas/assert/v2" @@ -31,3 +32,54 @@ func TestBox(t *testing.T) { Exec("docker", "compose", "-f", "echo-compose.yml", "down", "--rmi", "local"), ) } + +func TestSecretImportExport(t *testing.T) { + testImportExport(t, "secret") +} + +func TestConfigImportExport(t *testing.T) { + testImportExport(t, "config") +} + +func testImportExport(t *testing.T, object string) { + t.Helper() + + firstProjFile := "ftl-project.toml" + secondProjFile := "ftl-project-2.toml" + destinationFile := "exported.json" + + importPath, err := filepath.Abs("testdata/import.json") + assert.NoError(t, err) + + // use a pointer to keep track of the exported json so that i can be modified from within actions + blank := "" + exported := &blank + + RunWithoutController(t, "", + // duplicate project file in the temp directory + Exec("cp", firstProjFile, secondProjFile), + // import into first project file + Exec("ftl", object, "import", "--inline", "--config", firstProjFile, importPath), + + // export from first project file + ExecWithOutput("ftl", []string{object, "export", "--config", firstProjFile}, func(output string) { + *exported = output + + // make sure the exported json contains a value (otherwise the test could pass with the first import doing nothing) + assert.Contains(t, output, "test.one") + }), + + // import into second project file + // wrapped in a func to avoid capturing the initial valye of *exported + func(t testing.TB, ic TestContext) { + WriteFile(destinationFile, []byte(*exported))(t, ic) + Exec("ftl", object, "import", destinationFile, "--inline", "--config", secondProjFile)(t, ic) + }, + + // export from second project file + ExecWithOutput("ftl", []string{object, "export", "--config", secondProjFile}, func(output string) { + // check that both exported the same json + assert.Equal(t, *exported, output) + }), + ) +} diff --git a/cmd/ftl/testdata/import.json b/cmd/ftl/testdata/import.json new file mode 100644 index 0000000000..84be759fe4 --- /dev/null +++ b/cmd/ftl/testdata/import.json @@ -0,0 +1,5 @@ +{ + "test.one": 1, + "test.two": "a string", + "test2.three": {"key":"value"} +} \ No newline at end of file diff --git a/common/configuration/manager.go b/common/configuration/manager.go index 7490581f73..306464c687 100644 --- a/common/configuration/manager.go +++ b/common/configuration/manager.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "net/url" "os" "strings" @@ -75,6 +76,10 @@ func New[R Role](ctx context.Context, router Router[R], providers []Provider[R]) return m, nil } +func ProviderKeyForAccessor(accessor *url.URL) string { + return accessor.Scheme +} + // getData returns a data value for a configuration from the active providers. // The data can be unmarshalled from JSON. func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { @@ -89,7 +94,7 @@ func (m *Manager[R]) getData(ctx context.Context, ref Ref) ([]byte, error) { } else if err != nil { return nil, err } - provider, ok := m.providers[key.Scheme] + provider, ok := m.providers[ProviderKeyForAccessor(key)] if !ok { return nil, fmt.Errorf("no provider for scheme %q", key.Scheme) } diff --git a/frontend/src/protos/xyz/block/ftl/v1/ftl_pb.ts b/frontend/src/protos/xyz/block/ftl/v1/ftl_pb.ts index bacef1e933..7512612769 100644 --- a/frontend/src/protos/xyz/block/ftl/v1/ftl_pb.ts +++ b/frontend/src/protos/xyz/block/ftl/v1/ftl_pb.ts @@ -2663,6 +2663,11 @@ export class ListConfigRequest extends Message { */ includeValues?: boolean; + /** + * @generated from field: optional xyz.block.ftl.v1.ConfigProvider provider = 3; + */ + provider?: ConfigProvider; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -2673,6 +2678,7 @@ export class ListConfigRequest extends Message { static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "module", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 2, name: "include_values", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 3, name: "provider", kind: "enum", T: proto3.getEnumType(ConfigProvider), opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListConfigRequest { @@ -3014,6 +3020,11 @@ export class ListSecretsRequest extends Message { */ includeValues?: boolean; + /** + * @generated from field: optional xyz.block.ftl.v1.SecretProvider provider = 3; + */ + provider?: SecretProvider; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -3024,6 +3035,7 @@ export class ListSecretsRequest extends Message { static readonly fields: FieldList = proto3.util.newFieldList(() => [ { no: 1, name: "module", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 2, name: "include_values", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 3, name: "provider", kind: "enum", T: proto3.getEnumType(SecretProvider), opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): ListSecretsRequest { diff --git a/go-runtime/compile/compile_integration_test.go b/go-runtime/compile/compile_integration_test.go index 99980cbc87..5ea28de3d6 100644 --- a/go-runtime/compile/compile_integration_test.go +++ b/go-runtime/compile/compile_integration_test.go @@ -16,8 +16,9 @@ func TestNonExportedDecls(t *testing.T) { in.Deploy("echo"), in.CopyModule("notexportedverb"), in.ExpectError( - in.ExecWithOutput("ftl", "deploy", "notexportedverb"), - "call first argument must be a function but is an unresolved reference to echo.Echo, does it need to be exported?"), + in.ExecWithOutput("ftl", []string{"deploy", "notexportedverb"}, func(_ string) {}), + "call first argument must be a function but is an unresolved reference to echo.Echo, does it need to be exported?", + ), ) } @@ -29,7 +30,8 @@ func TestUndefinedExportedDecls(t *testing.T) { in.Deploy("echo"), in.CopyModule("undefinedverb"), in.ExpectError( - in.ExecWithOutput("ftl", "deploy", "undefinedverb"), - "call first argument must be a function but is an unresolved reference to echo.Undefined"), + in.ExecWithOutput("ftl", []string{"deploy", "undefinedverb"}, func(_ string) {}), + "call first argument must be a function but is an unresolved reference to echo.Undefined", + ), ) } diff --git a/integration/actions.go b/integration/actions.go index 33b8e90dd4..119593e04a 100644 --- a/integration/actions.go +++ b/integration/actions.go @@ -142,12 +142,14 @@ func ExecWithExpectedOutput(want string, cmd string, args ...string) Action { } // ExecWithOutput runs a command from the test working directory. -// The output is captured and is returned as part of the error. -func ExecWithOutput(cmd string, args ...string) Action { +// On success capture() is executed with the output +// On error, an error with the output is returned. +func ExecWithOutput(cmd string, args []string, capture func(output string)) Action { return func(t testing.TB, ic TestContext) { Infof("Executing: %s %s", cmd, shellquote.Join(args...)) output, err := ftlexec.Capture(ic, ic.workDir, cmd, args...) assert.NoError(t, err, "%s", string(output)) + capture(string(output)) } } @@ -253,6 +255,19 @@ func FileContent(path, expected string) Action { } } +// WriteFile writes a file to the working directory. +func WriteFile(path string, content []byte) Action { + return func(t testing.TB, ic TestContext) { + absPath := path + if !filepath.IsAbs(path) { + absPath = filepath.Join(ic.workDir, path) + } + Infof("Writing to %s", path) + err := os.WriteFile(absPath, content, 0600) + assert.NoError(t, err) + } +} + type Obj map[string]any // Call a verb. From 37423b43423877f151f6f314cd1295b5e2cc28e2 Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Mon, 8 Jul 2024 14:38:34 +1000 Subject: [PATCH 23/23] revert: "fix: use shared .ftl folder for stubs (#1843)" (#2003) This reverts commit 0a69db1b5db11d84538892ec525bddd00be0a5ef. Due to https://github.com/TBD54566975/ftl/issues/2001 --- .gitignore | 6 - Justfile | 2 +- buildengine/build.go | 8 +- buildengine/build_go.go | 4 +- buildengine/build_go_test.go | 249 ++++++++++++++++- buildengine/build_test.go | 36 +-- buildengine/deploy_test.go | 11 +- buildengine/discover_test.go | 14 +- buildengine/engine.go | 45 +--- buildengine/engine_test.go | 4 +- buildengine/stubs.go | 26 -- buildengine/stubs_test.go | 250 ------------------ cmd/ftl/cmd_box.go | 2 +- cmd/ftl/cmd_box_run.go | 5 +- cmd/ftl/cmd_build.go | 2 +- cmd/ftl/cmd_deploy.go | 5 +- cmd/ftl/cmd_dev.go | 2 +- cmd/ftl/cmd_init.go | 4 +- cmd/ftl/cmd_new.go | 10 +- common/moduleconfig/moduleconfig.go | 2 +- extensions/vscode/package-lock.json | 4 +- .../go/main/go.mod.tmpl | 2 +- .../go/main/main.go} | 0 .../compile/build-template/go.work.tmpl | 6 +- go-runtime/compile/build.go | 184 +++++-------- go-runtime/compile/devel.go | 5 - .../go/modules}/go.mod.tmpl | 4 +- .../external_module.go.tmpl | 32 +-- .../compile/external-module-template/go.mod | 3 + .../external-module-template/go.work.tmpl | 6 + .../compile/main-work-template/go.work.tmpl | 8 - go-runtime/compile/release.go | 11 - go-runtime/compile/schema_test.go | 8 +- .../{{ .Name | camel | lower }}/go.mod.tmpl | 4 +- integration/harness.go | 23 +- 35 files changed, 393 insertions(+), 594 deletions(-) delete mode 100644 buildengine/stubs.go delete mode 100644 buildengine/stubs_test.go rename go-runtime/compile/build-template/{.ftl.tmpl => _ftl.tmpl}/go/main/go.mod.tmpl (94%) rename go-runtime/compile/build-template/{.ftl.tmpl/go/main/main.go.tmpl => _ftl.tmpl/go/main/main.go} (100%) rename go-runtime/compile/external-module-template/{.ftl/go/modules/{{ .Module.Name }} => _ftl/go/modules}/go.mod.tmpl (80%) rename go-runtime/compile/external-module-template/{.ftl/go/modules/{{ .Module.Name }} => _ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}}/external_module.go.tmpl (66%) create mode 100644 go-runtime/compile/external-module-template/go.mod create mode 100644 go-runtime/compile/external-module-template/go.work.tmpl delete mode 100644 go-runtime/compile/main-work-template/go.work.tmpl diff --git a/.gitignore b/.gitignore index 31dcf44481..e54ce37176 100644 --- a/.gitignore +++ b/.gitignore @@ -20,15 +20,9 @@ examples/**/go.work examples/**/go.work.sum testdata/**/go.work testdata/**/go.work.sum - -# Leaving old _ftl for now to avoid old stuff getting checked in **/testdata/**/_ftl **/examples/**/_ftl **/examples/**/types.ftl.go -**/testdata/**/.ftl -**/examples/**/.ftl -/.ftl - buildengine/.gitignore go.work* junit*.xml diff --git a/Justfile b/Justfile index 50af62da26..0f6d1eb3f3 100644 --- a/Justfile +++ b/Justfile @@ -9,7 +9,7 @@ KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar" RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip" TIMESTAMP := `date +%s` SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto" -ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/compile/main-work-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" +ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" FRONTEND_OUT := "frontend/dist/index.html" EXTENSION_OUT := "extensions/vscode/dist/extension.js" PROTOS_IN := "backend/protos/xyz/block/ftl/v1/schema/schema.proto backend/protos/xyz/block/ftl/v1/console/console.proto backend/protos/xyz/block/ftl/v1/ftl.proto backend/protos/xyz/block/ftl/v1/schema/runtime.proto" diff --git a/buildengine/build.go b/buildengine/build.go index 3a50e0522e..2987223997 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -22,11 +22,11 @@ const BuildLockTimeout = time.Minute // Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func Build(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { - return buildModule(ctx, projectRootDir, sch, module, filesTransaction) +func Build(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { + return buildModule(ctx, sch, module, filesTransaction) } -func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { +func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { release, err := flock.Acquire(ctx, filepath.Join(module.Config.Dir, ".ftl.lock"), BuildLockTimeout) if err != nil { return err @@ -46,7 +46,7 @@ func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, switch module.Config.Language { case "go": - err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction) + err = buildGoModule(ctx, sch, module, filesTransaction) case "kotlin": err = buildKotlinModule(ctx, sch, module) case "rust": diff --git a/buildengine/build_go.go b/buildengine/build_go.go index 8eba01a747..ee8fb1c98b 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -8,8 +8,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGoModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { - if err := compile.Build(ctx, projectRootDir, module.Config.Dir, sch, transaction); err != nil { +func buildGoModule(ctx context.Context, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { + if err := compile.Build(ctx, module.Config.Dir, sch, transaction); err != nil { return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)} } return nil diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index cdfd9c8210..498e26e43b 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -11,6 +11,181 @@ import ( "github.com/TBD54566975/ftl/backend/schema" ) +func TestGenerateGoModule(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + sch := &schema.Schema{ + Modules: []*schema.Module{ + schema.Builtins(), + {Name: "other", Decls: []schema.Decl{ + &schema.Enum{ + Comments: []string{"This is an enum.", "", "It has 3 variants."}, + Name: "Color", + Export: true, + Type: &schema.String{}, + Variants: []*schema.EnumVariant{ + {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, + {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, + {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, + }, + }, + &schema.Enum{ + Name: "ColorInt", + Export: true, + Type: &schema.Int{}, + Variants: []*schema.EnumVariant{ + {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, + {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, + {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, + }, + }, + &schema.Enum{ + Comments: []string{"This is type enum."}, + Name: "TypeEnum", + Export: true, + Variants: []*schema.EnumVariant{ + {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, + {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, + }, + }, + &schema.Data{Name: "EchoRequest", Export: true}, + &schema.Data{ + Comments: []string{"This is an echo data response."}, + Name: "EchoResponse", Export: true}, + &schema.Verb{ + Name: "echo", + Export: true, + Request: &schema.Ref{Name: "EchoRequest"}, + Response: &schema.Ref{Name: "EchoResponse"}, + }, + &schema.Data{Name: "SinkReq", Export: true}, + &schema.Verb{ + Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, + Name: "sink", + Export: true, + Request: &schema.Ref{Name: "SinkReq"}, + Response: &schema.Unit{}, + }, + &schema.Data{Name: "SourceResp", Export: true}, + &schema.Verb{ + Name: "source", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Ref{Name: "SourceResp"}, + }, + &schema.Verb{ + Name: "nothing", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Unit{}, + }, + }}, + {Name: "test"}, + }, + } + expected := `// Code generated by FTL. DO NOT EDIT. + +package other + +import ( + "context" + + "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" +) + +var _ = context.Background + +// This is an enum. +// +// It has 3 variants. +// +//ftl:enum +type Color string +const ( + Red Color = "Red" + Blue Color = "Blue" + Green Color = "Green" +) + +//ftl:enum +type ColorInt int +const ( + RedInt ColorInt = 0 + BlueInt ColorInt = 1 + GreenInt ColorInt = 2 +) + +// This is type enum. +// +//ftl:enum +type TypeEnum interface { typeEnum() } + +type A int + +func (A) typeEnum() {} + +type B string + +func (B) typeEnum() {} + +type EchoRequest struct { +} + +// This is an echo data response. +// +type EchoResponse struct { +} + +//ftl:verb +func Echo(context.Context, EchoRequest) (EchoResponse, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} + +type SinkReq struct { +} + +// This is a sink verb. +// +// Here is another line for this comment! +// +//ftl:verb +func Sink(context.Context, SinkReq) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") +} + +type SourceResp struct { +} + +//ftl:verb +func Source(context.Context) (SourceResp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") +} + +//ftl:verb +func Nothing(context.Context) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") +} + +func init() { + reflection.Register( + reflection.SumType[TypeEnum]( + *new(A), + *new(B), + ), + ) +} +` + bctx := buildContext{ + moduleDir: "testdata/another", + buildDir: "_ftl", + sch: sch, + } + testBuild(t, bctx, "", []assertion{ + assertGeneratedModule("go/modules/other/external_module.go", expected), + }) +} + func TestGoBuildClearsBuildDir(t *testing.T) { if testing.Short() { t.SkipNow() @@ -23,22 +198,82 @@ func TestGoBuildClearsBuildDir(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/another", - buildDir: ".ftl", + buildDir: "_ftl", sch: sch, } testBuildClearsBuildDir(t, bctx) } +func TestMetadataImportsExcluded(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + sch := &schema.Schema{ + Modules: []*schema.Module{ + schema.Builtins(), + {Name: "test", Decls: []schema.Decl{ + &schema.Data{ + Comments: []string{"Request data type."}, + Name: "Req", Export: true}, + &schema.Data{Name: "Resp", Export: true}, + &schema.Verb{ + Comments: []string{"This is a verb."}, + Name: "call", + Export: true, + Request: &schema.Ref{Name: "Req"}, + Response: &schema.Ref{Name: "Resp"}, + Metadata: []schema.Metadata{ + &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, + }, + }, + }}, + }, + } + expected := `// Code generated by FTL. DO NOT EDIT. + +package test + +import ( + "context" +) + +var _ = context.Background + +// Request data type. +// +type Req struct { +} + +type Resp struct { +} + +// This is a verb. +// +//ftl:verb +func Call(context.Context, Req) (Resp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} +` + bctx := buildContext{ + moduleDir: "testdata/another", + buildDir: "_ftl", + sch: sch, + } + testBuild(t, bctx, "", []assertion{ + assertGeneratedModule("go/modules/test/external_module.go", expected), + }) +} + func TestExternalType(t *testing.T) { if testing.Short() { t.SkipNow() } bctx := buildContext{ moduleDir: "testdata/external", - buildDir: ".ftl", + buildDir: "_ftl", sch: &schema.Schema{}, } - testBuild(t, bctx, "", "unsupported external type", []assertion{ + testBuild(t, bctx, "unsupported external type", []assertion{ assertBuildProtoErrors( "unsupported external type \"time.Month\"", "unsupported type \"time.Month\" for field \"Month\"", @@ -67,10 +302,10 @@ func TestGoModVersion(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/highgoversion", - buildDir: ".ftl", + buildDir: "_ftl", sch: sch, } - testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), "", []assertion{}) + testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), []assertion{}) } func TestGeneratedTypeRegistry(t *testing.T) { @@ -110,10 +345,10 @@ func TestGeneratedTypeRegistry(t *testing.T) { assert.NoError(t, err) bctx := buildContext{ moduleDir: "testdata/other", - buildDir: ".ftl", + buildDir: "_ftl", sch: sch, } - testBuild(t, bctx, "", "", []assertion{ + testBuild(t, bctx, "", []assertion{ assertGeneratedMain(string(expected)), }) } diff --git a/buildengine/build_test.go b/buildengine/build_test.go index e29e244ea2..83a05b6c09 100644 --- a/buildengine/build_test.go +++ b/buildengine/build_test.go @@ -38,7 +38,6 @@ func (t *mockModifyFilesTransaction) End() error { func testBuild( t *testing.T, bctx buildContext, - expectedGeneratStubsErrMsg string, // emptystr if no error expected expectedBuildErrMsg string, // emptystr if no error expected assertions []assertion, ) { @@ -48,31 +47,12 @@ func testBuild( assert.NoError(t, err, "Error getting absolute path for module directory") module, err := LoadModule(abs) assert.NoError(t, err) - - projectRootDir := t.TempDir() - - configs := []moduleconfig.ModuleConfig{} - if bctx.moduleDir != "" { - config, err := moduleconfig.LoadModuleConfig(bctx.moduleDir) - assert.NoError(t, err, "Error loading project config") - configs = append(configs, config) - } - - // generate stubs to create the shared modules directory - err = GenerateStubs(ctx, projectRootDir, bctx.sch.Modules, configs) - if len(expectedGeneratStubsErrMsg) > 0 { + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) + if len(expectedBuildErrMsg) > 0 { assert.Error(t, err) - assert.Contains(t, err.Error(), expectedGeneratStubsErrMsg) + assert.Contains(t, err.Error(), expectedBuildErrMsg) } else { assert.NoError(t, err) - - err = Build(ctx, projectRootDir, bctx.sch, module, &mockModifyFilesTransaction{}) - if len(expectedBuildErrMsg) > 0 { - assert.Error(t, err) - assert.Contains(t, err.Error(), expectedBuildErrMsg) - } else { - assert.NoError(t, err) - } } for _, a := range assertions { @@ -90,16 +70,10 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { abs, err := filepath.Abs(bctx.moduleDir) assert.NoError(t, err, "Error getting absolute path for module directory") - projectRoot := t.TempDir() - - // generate stubs to create the shared modules directory - err = GenerateStubs(ctx, projectRoot, bctx.sch.Modules, []moduleconfig.ModuleConfig{{Dir: bctx.moduleDir}}) - assert.NoError(t, err) - // build to generate the build directory module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // create a temporary file in the build directory @@ -111,7 +85,7 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { // build to clear the old build directory module, err = LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // ensure the temporary file was removed diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index 3550668d26..029f2d33af 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -10,7 +10,6 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/sha256" ) @@ -72,17 +71,11 @@ func TestDeploy(t *testing.T) { module, err := LoadModule(modulePath) assert.NoError(t, err) - projectRootDir := t.TempDir() - - // generate stubs to create the shared modules directory - err = GenerateStubs(ctx, projectRootDir, sch.Modules, []moduleconfig.ModuleConfig{module.Config}) - assert.NoError(t, err) - // Build first to make sure the files are there. - err = Build(ctx, projectRootDir, sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) - sum, err := sha256.SumFile(modulePath + "/.ftl/main") + sum, err := sha256.SumFile(modulePath + "/_ftl/main") assert.NoError(t, err) client := &mockDeployClient{ diff --git a/buildengine/discover_test.go b/buildengine/discover_test.go index 0aec6ccc97..b63d7d3416 100644 --- a/buildengine/discover_test.go +++ b/buildengine/discover_test.go @@ -22,7 +22,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "alpha", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -35,7 +35,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "another", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -48,7 +48,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle1", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -61,7 +61,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle2", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -100,7 +100,7 @@ func TestDiscoverModules(t *testing.T) { Deploy: []string{ "main", }, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{ @@ -140,7 +140,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "highgoversion", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -153,7 +153,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "other", Deploy: []string{"main"}, - DeployDir: ".ftl", + DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, diff --git a/buildengine/engine.go b/buildengine/engine.go index dbc109e159..b9a7b4215b 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -21,7 +21,6 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/rpc" "github.com/TBD54566975/ftl/internal/slices" @@ -66,7 +65,6 @@ type Listener interface { type Engine struct { client ftlv1connect.ControllerServiceClient moduleMetas *xsync.MapOf[string, moduleMeta] - projectRoot string moduleDirs []string watcher *Watcher controllerSchema *xsync.MapOf[string, *schema.Module] @@ -99,11 +97,10 @@ func WithListener(listener Listener) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, projectRoot string, moduleDirs []string, options ...Option) (*Engine, error) { +func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, - projectRoot: projectRoot, moduleDirs: moduleDirs, moduleMetas: xsync.NewMapOf[string, moduleMeta](), watcher: NewWatcher(), @@ -569,21 +566,6 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, } errCh := make(chan error, 1024) for _, group := range topology { - groupSchemas := map[string]*schema.Module{} - metas, err := e.gatherGroupSchemas(builtModules, group, groupSchemas) - if err != nil { - return err - } - - moduleConfigs := make([]moduleconfig.ModuleConfig, len(metas)) - for i, meta := range metas { - moduleConfigs[i] = meta.module.Config - } - err = GenerateStubs(ctx, e.projectRoot, maps.Values(groupSchemas), moduleConfigs) - if err != nil { - return err - } - // Collect schemas to be inserted into "built" map for subsequent groups. schemas := make(chan *schema.Module, len(group)) @@ -682,7 +664,7 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ if e.listener != nil { e.listener.OnBuildStarted(meta.module) } - err := Build(ctx, e.projectRoot, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) + err := Build(ctx, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) if err != nil { return err } @@ -695,29 +677,6 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ return nil } -// Construct a combined schema for a group of modules and their transitive dependencies. -func (e *Engine) gatherGroupSchemas( - moduleSchemas map[string]*schema.Module, - group []string, - out map[string]*schema.Module, -) ([]moduleMeta, error) { - var metas []moduleMeta - for _, module := range group { - if module == "builtin" { - continue // Skip the builtin module - } - - meta, ok := e.moduleMetas.Load(module) - if ok { - metas = append(metas, meta) - if err := e.gatherSchemas(moduleSchemas, meta.module, out); err != nil { - return nil, err - } - } - } - return metas, nil -} - // Construct a combined schema for a module and its transitive dependencies. func (e *Engine) gatherSchemas( moduleSchemas map[string]*schema.Module, diff --git a/buildengine/engine_test.go b/buildengine/engine_test.go index a44ab8768a..73a5211349 100644 --- a/buildengine/engine_test.go +++ b/buildengine/engine_test.go @@ -16,7 +16,7 @@ func TestEngine(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/alpha", "testdata/other", "testdata/another"}) + engine, err := buildengine.New(ctx, nil, []string{"testdata/alpha", "testdata/other", "testdata/another"}) assert.NoError(t, err) defer engine.Close() @@ -64,7 +64,7 @@ func TestCycleDetection(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/depcycle1", "testdata/depcycle2"}) + engine, err := buildengine.New(ctx, nil, []string{"testdata/depcycle1", "testdata/depcycle2"}) assert.NoError(t, err) defer engine.Close() diff --git a/buildengine/stubs.go b/buildengine/stubs.go deleted file mode 100644 index 911d00d104..0000000000 --- a/buildengine/stubs.go +++ /dev/null @@ -1,26 +0,0 @@ -package buildengine - -import ( - "context" - "fmt" - - "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" - "github.com/TBD54566975/ftl/go-runtime/compile" -) - -// GenerateStubs generates stubs for the given modules. -// -// Currently, only Go stubs are supported. Kotlin and other language stubs can be added in the future. -func GenerateStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { - return generateGoStubs(ctx, projectRoot, modules, moduleConfigs) -} - -func generateGoStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { - sch := &schema.Schema{Modules: modules} - err := compile.GenerateStubsForModules(ctx, projectRoot, moduleConfigs, sch) - if err != nil { - return fmt.Errorf("failed to generate Go stubs: %w", err) - } - return nil -} diff --git a/buildengine/stubs_test.go b/buildengine/stubs_test.go deleted file mode 100644 index c4c1efe5b5..0000000000 --- a/buildengine/stubs_test.go +++ /dev/null @@ -1,250 +0,0 @@ -package buildengine - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" - "github.com/TBD54566975/ftl/internal/log" - "github.com/alecthomas/assert/v2" -) - -func TestGenerateGoStubs(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - modules := []*schema.Module{ - schema.Builtins(), - {Name: "other", Decls: []schema.Decl{ - &schema.Enum{ - Comments: []string{"This is an enum.", "", "It has 3 variants."}, - Name: "Color", - Export: true, - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, - }, - }, - &schema.Enum{ - Name: "ColorInt", - Export: true, - Type: &schema.Int{}, - Variants: []*schema.EnumVariant{ - {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, - {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, - {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, - }, - }, - &schema.Enum{ - Comments: []string{"This is type enum."}, - Name: "TypeEnum", - Export: true, - Variants: []*schema.EnumVariant{ - {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, - {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, - }, - }, - &schema.Data{Name: "EchoRequest", Export: true}, - &schema.Data{ - Comments: []string{"This is an echo data response."}, - Name: "EchoResponse", Export: true}, - &schema.Verb{ - Name: "echo", - Export: true, - Request: &schema.Ref{Name: "EchoRequest"}, - Response: &schema.Ref{Name: "EchoResponse"}, - }, - &schema.Data{Name: "SinkReq", Export: true}, - &schema.Verb{ - Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, - Name: "sink", - Export: true, - Request: &schema.Ref{Name: "SinkReq"}, - Response: &schema.Unit{}, - }, - &schema.Data{Name: "SourceResp", Export: true}, - &schema.Verb{ - Name: "source", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Ref{Name: "SourceResp"}, - }, - &schema.Verb{ - Name: "nothing", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Unit{}, - }, - }}, - {Name: "test"}, - } - - expected := `// Code generated by FTL. DO NOT EDIT. - -package other - -import ( - "context" - - "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" -) - -var _ = context.Background - -// This is an enum. -// -// It has 3 variants. -// -//ftl:enum -type Color string -const ( - Red Color = "Red" - Blue Color = "Blue" - Green Color = "Green" -) - -//ftl:enum -type ColorInt int -const ( - RedInt ColorInt = 0 - BlueInt ColorInt = 1 - GreenInt ColorInt = 2 -) - -// This is type enum. -// -//ftl:enum -type TypeEnum interface { typeEnum() } - -type A int - -func (A) typeEnum() {} - -type B string - -func (B) typeEnum() {} - -type EchoRequest struct { -} - -// This is an echo data response. -// -type EchoResponse struct { -} - -//ftl:verb -func Echo(context.Context, EchoRequest) (EchoResponse, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} - -type SinkReq struct { -} - -// This is a sink verb. -// -// Here is another line for this comment! -// -//ftl:verb -func Sink(context.Context, SinkReq) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") -} - -type SourceResp struct { -} - -//ftl:verb -func Source(context.Context) (SourceResp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") -} - -//ftl:verb -func Nothing(context.Context) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") -} - -func init() { - reflection.Register( - reflection.SumType[TypeEnum]( - *new(A), - *new(B), - ), - ) -} -` - - ctx := log.ContextWithNewDefaultLogger(context.Background()) - projectRoot := t.TempDir() - err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) - assert.NoError(t, err) - - generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/other/external_module.go") - fileContent, err := os.ReadFile(generatedPath) - assert.NoError(t, err) - assert.Equal(t, expected, string(fileContent)) -} - -func TestMetadataImportsExcluded(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - modules := []*schema.Module{ - schema.Builtins(), - {Name: "test", Decls: []schema.Decl{ - &schema.Data{ - Comments: []string{"Request data type."}, - Name: "Req", Export: true}, - &schema.Data{Name: "Resp", Export: true}, - &schema.Verb{ - Comments: []string{"This is a verb."}, - Name: "call", - Export: true, - Request: &schema.Ref{Name: "Req"}, - Response: &schema.Ref{Name: "Resp"}, - Metadata: []schema.Metadata{ - &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, - }, - }, - }}, - } - - expected := `// Code generated by FTL. DO NOT EDIT. - -package test - -import ( - "context" -) - -var _ = context.Background - -// Request data type. -// -type Req struct { -} - -type Resp struct { -} - -// This is a verb. -// -//ftl:verb -func Call(context.Context, Req) (Resp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} -` - ctx := log.ContextWithNewDefaultLogger(context.Background()) - projectRoot := t.TempDir() - err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) - assert.NoError(t, err) - - generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/test/external_module.go") - fileContent, err := os.ReadFile(generatedPath) - assert.NoError(t, err) - assert.Equal(t, expected, string(fileContent)) -} diff --git a/cmd/ftl/cmd_box.go b/cmd/ftl/cmd_box.go index 887eefb763..d09214c887 100644 --- a/cmd/ftl/cmd_box.go +++ b/cmd/ftl/cmd_box.go @@ -123,7 +123,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_box_run.go b/cmd/ftl/cmd_box_run.go index 0565edd540..0f9cb08b52 100644 --- a/cmd/ftl/cmd_box_run.go +++ b/cmd/ftl/cmd_box_run.go @@ -17,7 +17,6 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/buildengine" - "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" @@ -34,7 +33,7 @@ type boxRunCmd struct { ControllerTimeout time.Duration `help:"Timeout for Controller start." default:"30s"` } -func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { +func (b *boxRunCmd) Run(ctx context.Context) error { conn, err := databasetesting.CreateForDevel(ctx, b.DSN, b.Recreate) if err != nil { return fmt.Errorf("failed to create database: %w", err) @@ -75,7 +74,7 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er return fmt.Errorf("controller failed to start: %w", err) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), []string{b.Dir}) + engine, err := buildengine.New(ctx, client, []string{b.Dir}) if err != nil { return fmt.Errorf("failed to create build engine: %w", err) } diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index a3f13869b2..35e826e740 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -22,7 +22,7 @@ func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServic if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index a50c7f25fc..0ecd47a6ed 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -5,7 +5,6 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/buildengine" - "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/rpc" ) @@ -16,9 +15,9 @@ type deployCmd struct { NoWait bool `help:"Do not wait for deployment to complete." default:"false"` } -func (d *deployCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { +func (d *deployCmd) Run(ctx context.Context) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go index e596d84fa3..2741faecfe 100644 --- a/cmd/ftl/cmd_dev.go +++ b/cmd/ftl/cmd_dev.go @@ -89,7 +89,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error }) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, opts...) + engine, err := buildengine.New(ctx, client, d.Dirs, opts...) if err != nil { return err } diff --git a/cmd/ftl/cmd_init.go b/cmd/ftl/cmd_init.go index b2cafb64e3..b394639797 100644 --- a/cmd/ftl/cmd_init.go +++ b/cmd/ftl/cmd_init.go @@ -86,7 +86,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { scanner := bufio.NewScanner(f) for scanner.Scan() { - if strings.TrimSpace(scanner.Text()) == "**/.ftl" { + if strings.TrimSpace(scanner.Text()) == "**/_ftl" { return nil } } @@ -96,7 +96,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { } // append if not already present - if _, err = f.WriteString("**/.ftl\n"); err != nil { + if _, err = f.WriteString("**/_ftl\n"); err != nil { return err } diff --git a/cmd/ftl/cmd_new.go b/cmd/ftl/cmd_new.go index 1afa32ad1f..831364a03c 100644 --- a/cmd/ftl/cmd_new.go +++ b/cmd/ftl/cmd_new.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "regexp" - "runtime" "strings" "github.com/TBD54566975/scaffolder" @@ -31,10 +30,9 @@ type newCmd struct { } type newGoCmd struct { - Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` - Dir string `arg:"" help:"Directory to initialize the module in."` - Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` - GoVersion string + Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` + Dir string `arg:"" help:"Directory to initialize the module in."` + Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` } type newKotlinCmd struct { @@ -60,8 +58,6 @@ func (i newGoCmd) Run(ctx context.Context) error { logger := log.FromContext(ctx) logger.Debugf("Creating FTL Go module %q in %s", name, path) - - i.GoVersion = runtime.Version()[2:] if err := scaffold(ctx, config.Hermit, goruntime.Files(), i.Dir, i, scaffolder.Exclude("^go.mod$")); err != nil { return err } diff --git a/common/moduleconfig/moduleconfig.go b/common/moduleconfig/moduleconfig.go index 990e96b732..dddad51340 100644 --- a/common/moduleconfig/moduleconfig.go +++ b/common/moduleconfig/moduleconfig.go @@ -133,7 +133,7 @@ func setConfigDefaults(moduleDir string, config *ModuleConfig) error { case "go": if config.DeployDir == "" { - config.DeployDir = ".ftl" + config.DeployDir = "_ftl" } if len(config.Deploy) == 0 { config.Deploy = []string{"main"} diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index a539cf6ee4..dbc72146ec 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "ftl", - "version": "0.0.0", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ftl", - "version": "0.0.0", + "version": "0.1.2", "dependencies": { "lookpath": "^1.2.2", "semver": "^7.6.0", diff --git a/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl b/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl similarity index 94% rename from go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl rename to go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl index df0b6eca75..56b07af1f7 100644 --- a/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl +++ b/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl b/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go similarity index 100% rename from go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl rename to go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go diff --git a/go-runtime/compile/build-template/go.work.tmpl b/go-runtime/compile/build-template/go.work.tmpl index cbd6143cf0..683cd1d340 100644 --- a/go-runtime/compile/build-template/go.work.tmpl +++ b/go-runtime/compile/build-template/go.work.tmpl @@ -2,8 +2,6 @@ go {{ .GoVersion }} use ( . -{{- range .SharedModulesPaths }} - {{ . }} -{{- end }} - .ftl/go/main + _ftl/go/modules + _ftl/go/main ) diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 4b37742949..4a234a356d 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -3,6 +3,7 @@ package compile import ( "context" "fmt" + "maps" "os" "path" "path/filepath" @@ -14,7 +15,7 @@ import ( "unicode" sets "github.com/deckarep/golang-set/v2" - "golang.org/x/exp/maps" + gomaps "golang.org/x/exp/maps" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" @@ -34,16 +35,12 @@ import ( "github.com/TBD54566975/ftl/internal/reflect" ) -type MainWorkContext struct { - GoVersion string - SharedModulesPaths []string -} - type ExternalModuleContext struct { + ModuleDir string *schema.Schema GoVersion string FTLVersion string - Module *schema.Module + Main string Replacements []*modfile.Replace } @@ -56,14 +53,13 @@ type goVerb struct { } type mainModuleContext struct { - GoVersion string - FTLVersion string - Name string - SharedModulesPaths []string - Verbs []goVerb - Replacements []*modfile.Replace - SumTypes []goSumType - LocalSumTypes []goSumType + GoVersion string + FTLVersion string + Name string + Verbs []goVerb + Replacements []*modfile.Replace + SumTypes []goSumType + LocalSumTypes []goSumType } type goSumType struct { @@ -83,14 +79,25 @@ type ModifyFilesTransaction interface { End() error } -const buildDirName = ".ftl" +func (b ExternalModuleContext) NonMainModules() []*schema.Module { + modules := make([]*schema.Module, 0, len(b.Modules)) + for _, module := range b.Modules { + if module.Name == b.Main { + continue + } + modules = append(modules, module) + } + return modules +} + +const buildDirName = "_ftl" func buildDir(moduleDir string) string { return filepath.Join(moduleDir, buildDirName) } // Build the given module. -func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { +func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { if err := filesTransaction.Begin(); err != nil { return err } @@ -123,27 +130,19 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc funcs := maps.Clone(scaffoldFuncs) - buildDir := buildDir(moduleDir) - err = os.MkdirAll(buildDir, 0750) - if err != nil { - return fmt.Errorf("failed to create build directory: %w", err) - } - - var sharedModulesPaths []string - for _, mod := range sch.Modules { - if mod.Name == config.Module { - continue - } - sharedModulesPaths = append(sharedModulesPaths, filepath.Join(projectRootDir, buildDirName, "go", "modules", mod.Name)) - } - - if err := internal.ScaffoldZip(mainWorkTemplateFiles(), moduleDir, MainWorkContext{ - GoVersion: goModVersion, - SharedModulesPaths: sharedModulesPaths, - }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { - return fmt.Errorf("failed to scaffold zip: %w", err) + logger.Debugf("Generating external modules") + if err := generateExternalModules(ExternalModuleContext{ + ModuleDir: moduleDir, + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Schema: sch, + Main: config.Module, + Replacements: replacements, + }); err != nil { + return fmt.Errorf("failed to generate external modules: %w", err) } + buildDir := buildDir(moduleDir) logger.Debugf("Extracting schema") result, err := ExtractModuleSchema(config.Dir, sch) if err != nil { @@ -187,14 +186,13 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc goVerbs = append(goVerbs, goverb) } if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{ - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Name: result.Module.Name, - SharedModulesPaths: sharedModulesPaths, - Verbs: goVerbs, - Replacements: replacements, - SumTypes: getSumTypes(result.Module, sch, result.NativeNames), - LocalSumTypes: getLocalSumTypes(result.Module), + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Name: result.Module.Name, + Verbs: goVerbs, + Replacements: replacements, + SumTypes: getSumTypes(result.Module, sch, result.NativeNames), + LocalSumTypes: getLocalSumTypes(result.Module), }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -202,7 +200,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc logger.Debugf("Tidying go.mod files") wg, wgctx := errgroup.WithContext(ctx) wg.Go(func() error { - if err := exec.Command(wgctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { + if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", moduleDir, err) } return filesTransaction.ModifiedFiles(filepath.Join(moduleDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) @@ -214,6 +212,13 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc } return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) + modulesDir := filepath.Join(buildDir, "go", "modules") + wg.Go(func() error { + if err := exec.Command(wgctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { + return fmt.Errorf("%s: failed to tidy go.mod: %w", modulesDir, err) + } + return filesTransaction.ModifiedFiles(filepath.Join(modulesDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) + }) if err := wg.Wait(); err != nil { return err } @@ -222,16 +227,10 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Sc return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } -func GenerateStubsForModules(ctx context.Context, projectRoot string, moduleConfigs []moduleconfig.ModuleConfig, sch *schema.Schema) error { - logger := log.FromContext(ctx) - logger.Debugf("Generating stubs for modules") - - sharedFtlDir := filepath.Join(projectRoot, buildDirName) - - // Wipe the modules directory to ensure we don't have any stale modules. - err := os.RemoveAll(sharedFtlDir) +func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error { + goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod")) if err != nil { - return fmt.Errorf("failed to remove %s: %w", sharedFtlDir, err) + return fmt.Errorf("failed to propagate replacements for library %q: %w", dir, err) } ftlVersion := "" @@ -239,64 +238,27 @@ func GenerateStubsForModules(ctx context.Context, projectRoot string, moduleConf ftlVersion = ftl.Version } - for _, module := range sch.Modules { - var moduleConfig *moduleconfig.ModuleConfig - for _, mc := range moduleConfigs { - mcCopy := mc - if mc.Module == module.Name { - moduleConfig = &mcCopy - break - } - } - - var goModVersion string - var replacements []*modfile.Replace - - // If there's no module config, use the go.mod file for the first config we find. - if moduleConfig == nil { - if len(moduleConfigs) > 0 { - _, goModVersion, err = updateGoModule(filepath.Join(moduleConfigs[0].Dir, "go.mod")) - if err != nil { - return err - } - } else { - // The best we can do here if we don't have a module to read from is to use the current Go version. - goModVersion = runtime.Version()[2:] - } - - replacements = []*modfile.Replace{} - } else { - replacements, goModVersion, err = updateGoModule(filepath.Join(moduleConfig.Dir, "go.mod")) - if err != nil { - return err - } - } - - goVersion := runtime.Version()[2:] - if semver.Compare("v"+goVersion, "v"+goModVersion) < 0 { - return fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion) - } - - context := ExternalModuleContext{ - Schema: sch, - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Module: module, - Replacements: replacements, - } - - funcs := maps.Clone(scaffoldFuncs) - err = internal.ScaffoldZip(externalModuleTemplateFiles(), projectRoot, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) - if err != nil { - return fmt.Errorf("failed to scaffold zip: %w", err) - } + return generateExternalModules(ExternalModuleContext{ + ModuleDir: dir, + GoVersion: goModFile.Go.Version, + FTLVersion: ftlVersion, + Schema: schema, + Replacements: replacements, + }) +} - modulesDir := filepath.Join(sharedFtlDir, "go", "modules", module.Name) - if err := exec.Command(ctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { - return fmt.Errorf("failed to tidy go.mod: %w", err) - } +func generateExternalModules(context ExternalModuleContext) error { + // Wipe the modules directory to ensure we don't have any stale modules. + err := os.RemoveAll(filepath.Join(buildDir(context.ModuleDir), "go", "modules")) + if err != nil { + return fmt.Errorf("could not remove old external modules: %w", err) } + funcs := maps.Clone(scaffoldFuncs) + err = internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) + if err != nil { + return fmt.Errorf("failed to scaffold external modules: %w", err) + } return nil } @@ -613,7 +575,7 @@ func getLocalSumTypes(module *schema.Module) []goSumType { } } } - out := maps.Values(sumTypes) + out := gomaps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) @@ -656,7 +618,7 @@ func getSumTypes(module *schema.Module, sch *schema.Schema, nativeNames NativeNa Variants: variants, } } - out := maps.Values(sumTypes) + out := gomaps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) diff --git a/go-runtime/compile/devel.go b/go-runtime/compile/devel.go index 345309da66..c6f2f6270c 100644 --- a/go-runtime/compile/devel.go +++ b/go-runtime/compile/devel.go @@ -8,14 +8,9 @@ import ( "github.com/TBD54566975/ftl/internal" ) -func mainWorkTemplateFiles() *zip.Reader { - return internal.ZipRelativeToCaller("main-work-template") -} - func externalModuleTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("external-module-template") } - func buildTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("build-template") } diff --git a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl b/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl similarity index 80% rename from go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl rename to go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl index be3d760791..2f1d05e47d 100644 --- a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl +++ b/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl @@ -1,4 +1,4 @@ -module ftl/{{ .Module.Name }} +module ftl go {{ .GoVersion }} @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl b/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl similarity index 66% rename from go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl rename to go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl index 657e8f20ab..a7c7c1b2de 100644 --- a/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl +++ b/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl @@ -1,13 +1,13 @@ // Code generated by FTL. DO NOT EDIT. -package {{.Module.Name}} +package {{.Name}} import ( "context" -{{- range $import, $alias := (.Module | imports)}} +{{- range $import, $alias := (.|imports)}} {{if $alias}}{{$alias}} {{end}}"{{$import}}" {{- end}} -{{- $sumTypes := (.Module | sumTypes)}} +{{- $sumTypes := $ | sumTypes}} {{- if $sumTypes}} "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" @@ -16,18 +16,18 @@ import ( var _ = context.Background -{{- range .Module.Decls }} +{{- range .Decls }} {{- if .IsExported}} {{if .Comments}} {{.Comments|comment -}} // {{- end}} {{- if is "Topic" .}} -var {{.Name|title}} = ftl.Topic[{{type $.Module .Event}}]("{{.Name}}") +var {{.Name|title}} = ftl.Topic[{{type $ .Event}}]("{{.Name}}") {{- else if and (is "Enum" .) .IsValueEnum}} {{- $enumName := .Name}} //ftl:enum -type {{.Name|title}} {{type $.Module .Type}} +type {{.Name|title}} {{type $ .Type}} const ( {{- range .Variants }} {{.Name|title}} {{$enumName}} = {{.Value|value}} @@ -38,14 +38,14 @@ const ( {{$enumInterfaceFuncName := enumInterfaceFunc . -}} type {{.Name|title}} interface { {{$enumInterfaceFuncName}}() } {{- range .Variants }} -{{if (or (basicType $.Module .) (isStandaloneEnumVariant .))}} -type {{.Name|title}} {{type $.Module .Value.Value}} +{{if (or (basicType $ .) (isStandaloneEnumVariant .))}} +type {{.Name|title}} {{type $ .Value.Value}} {{end}} func ({{.Name|title}}) {{$enumInterfaceFuncName}}() {} {{- end}} {{- else if is "TypeAlias" .}} //ftl:typealias -type {{.Name|title}} {{type $.Module .Type}} +type {{.Name|title}} {{type $ .Type}} {{- else if is "Data" .}} type {{.Name|title}} {{- if .TypeParameters}}[ @@ -54,25 +54,25 @@ type {{.Name|title}} {{- end -}} ]{{- end}} struct { {{- range .Fields}} - {{.Name|title}} {{type $.Module .Type}} `json:"{{.Name}}"` + {{.Name|title}} {{type $ .Type}} `json:"{{.Name}}"` {{- end}} } {{- else if is "Verb" .}} //ftl:verb -{{- if and (eq (type $.Module .Request) "ftl.Unit") (eq (type $.Module .Response) "ftl.Unit")}} +{{- if and (eq (type $ .Request) "ftl.Unit") (eq (type $ .Response) "ftl.Unit")}} func {{.Name|title}}(context.Context) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") } -{{- else if eq (type $.Module .Request) "ftl.Unit"}} -func {{.Name|title}}(context.Context) ({{type $.Module .Response}}, error) { +{{- else if eq (type $ .Request) "ftl.Unit"}} +func {{.Name|title}}(context.Context) ({{type $ .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") } -{{- else if eq (type $.Module .Response) "ftl.Unit"}} -func {{.Name|title}}(context.Context, {{type $.Module .Request}}) error { +{{- else if eq (type $ .Response) "ftl.Unit"}} +func {{.Name|title}}(context.Context, {{type $ .Request}}) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") } {{- else}} -func {{.Name|title}}(context.Context, {{type $.Module .Request}}) ({{type $.Module .Response}}, error) { +func {{.Name|title}}(context.Context, {{type $ .Request}}) ({{type $ .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") } {{- end}} diff --git a/go-runtime/compile/external-module-template/go.mod b/go-runtime/compile/external-module-template/go.mod new file mode 100644 index 0000000000..8da40da65a --- /dev/null +++ b/go-runtime/compile/external-module-template/go.mod @@ -0,0 +1,3 @@ +module exclude + +go 1.22.2 diff --git a/go-runtime/compile/external-module-template/go.work.tmpl b/go-runtime/compile/external-module-template/go.work.tmpl new file mode 100644 index 0000000000..204b39b25e --- /dev/null +++ b/go-runtime/compile/external-module-template/go.work.tmpl @@ -0,0 +1,6 @@ +go {{ .GoVersion }} + +use ( + . + _ftl/go/modules +) diff --git a/go-runtime/compile/main-work-template/go.work.tmpl b/go-runtime/compile/main-work-template/go.work.tmpl deleted file mode 100644 index 1dbed607a3..0000000000 --- a/go-runtime/compile/main-work-template/go.work.tmpl +++ /dev/null @@ -1,8 +0,0 @@ -go {{ .GoVersion }} - -use ( - . -{{- range .SharedModulesPaths }} - {{ . }} -{{- end }} -) diff --git a/go-runtime/compile/release.go b/go-runtime/compile/release.go index a258dd2170..8b909688cd 100644 --- a/go-runtime/compile/release.go +++ b/go-runtime/compile/release.go @@ -8,23 +8,12 @@ import ( _ "embed" ) -//go:embed main-work-template.zip -var mainWorkTemplateBytes []byte - //go:embed external-module-template.zip var externalModuleTemplateBytes []byte //go:embed build-template.zip var buildTemplateBytes []byte -func mainWorkTemplateFiles() *zip.Reader { - zr, err := zip.NewReader(bytes.NewReader(mainWorkTemplateBytes), int64(len(mainWorkTemplateBytes))) - if err != nil { - panic(err) - } - return zr -} - func externalModuleTemplateFiles() *zip.Reader { zr, err := zip.NewReader(bytes.NewReader(externalModuleTemplateBytes), int64(len(externalModuleTemplateBytes))) if err != nil { diff --git a/go-runtime/compile/schema_test.go b/go-runtime/compile/schema_test.go index abbc0d83db..8c974259bf 100644 --- a/go-runtime/compile/schema_test.go +++ b/go-runtime/compile/schema_test.go @@ -56,7 +56,7 @@ func TestExtractModuleSchema(t *testing.T) { assert.NoError(t, err) actual := schema.Normalise(r.Module) expected := `module one { - config configValue one.Config + config configValue one.Config secret secretValue String database postgres testDb @@ -184,9 +184,6 @@ func TestExtractModuleSchemaTwo(t *testing.T) { if testing.Short() { t.SkipNow() } - - assert.NoError(t, prebuildTestModule(t, "testdata/two")) - r, err := ExtractModuleSchema("testdata/two", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, r.Errors, nil) @@ -368,9 +365,6 @@ func TestExtractModulePubSub(t *testing.T) { if testing.Short() { t.SkipNow() } - - assert.NoError(t, prebuildTestModule(t, "testdata/pubsub")) - r, err := ExtractModuleSchema("testdata/pubsub", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, nil, r.Errors, "expected no schema errors") diff --git a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl index 7d6b63f1d2..372814bd3b 100644 --- a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl +++ b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl @@ -1,9 +1,9 @@ module ftl/{{ .Name }} -go {{ .GoVersion }} +go 1.21 require github.com/TBD54566975/ftl latest {{- range $old, $new := .Replace }} replace {{ $old }} => {{ $new }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/integration/harness.go b/integration/harness.go index 6cb0af3369..d4c55fc955 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -17,7 +17,6 @@ import ( "connectrpc.com/connect" "github.com/alecthomas/assert/v2" "github.com/alecthomas/types/optional" - "github.com/otiai10/copy" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" @@ -74,23 +73,11 @@ func run(t *testing.T, ftlConfigPath string, startController bool, actions ...Ac assert.True(t, ok) if ftlConfigPath != "" { - ftlConfigPath = filepath.Join(cwd, "testdata", "go", ftlConfigPath) - projectPath := filepath.Join(tmpDir, "ftl-project.toml") - - // Copy the specified FTL config to the temporary directory. - err = copy.Copy(ftlConfigPath, projectPath) - if err == nil { - t.Setenv("FTL_CONFIG", projectPath) - } else { - // Use a path into the testdata directory instead of one relative to - // tmpDir. Otherwise we have a chicken and egg situation where the config - // can't be loaded until the module is copied over, and the config itself - // is used by FTL during startup. - // Some tests still rely on this behavior, so we can't remove it entirely. - t.Logf("Failed to copy %s to %s: %s", ftlConfigPath, projectPath, err) - t.Setenv("FTL_CONFIG", ftlConfigPath) - } - + // Use a path into the testdata directory instead of one relative to + // tmpDir. Otherwise we have a chicken and egg situation where the config + // can't be loaded until the module is copied over, and the config itself + // is used by FTL during startup. + t.Setenv("FTL_CONFIG", filepath.Join(cwd, "testdata", "go", ftlConfigPath)) } else { err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644) assert.NoError(t, err)