diff --git a/backend/controller/admin/admin.go b/backend/controller/admin/admin.go index 4d5928c8a7..f360d8fc12 100644 --- a/backend/controller/admin/admin.go +++ b/backend/controller/admin/admin.go @@ -6,12 +6,10 @@ import ( "fmt" "connectrpc.com/connect" - "github.com/alecthomas/types/optional" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/go-runtime/encoding" - "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/configuration" "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/configuration/providers" @@ -23,25 +21,22 @@ type AdminService struct { schr SchemaRetriever cm *manager.Manager[configuration.Configuration] sm *manager.Manager[configuration.Secrets] - // bindAllocator needs to be set for local client to retrieve schemas from disk using language plugins - bindAllocator optional.Option[*bind.BindAllocator] } var _ ftlv1connect.AdminServiceHandler = (*AdminService)(nil) type SchemaRetriever interface { // BindAllocator is required if the schema is retrieved from disk using language plugins - GetActiveSchema(ctx context.Context, bindAllocator optional.Option[*bind.BindAllocator]) (*schema.Schema, error) + GetActiveSchema(ctx context.Context) (*schema.Schema, error) } // NewAdminService creates a new AdminService. // bindAllocator is optional and should be set if a local client is to be used that accesses schema from disk using language plugins. -func NewAdminService(cm *manager.Manager[configuration.Configuration], sm *manager.Manager[configuration.Secrets], schr SchemaRetriever, bindAllocator optional.Option[*bind.BindAllocator]) *AdminService { +func NewAdminService(cm *manager.Manager[configuration.Configuration], sm *manager.Manager[configuration.Secrets], schr SchemaRetriever) *AdminService { return &AdminService{ - schr: schr, - cm: cm, - sm: sm, - bindAllocator: bindAllocator, + schr: schr, + cm: cm, + sm: sm, } } @@ -254,7 +249,7 @@ func (s *AdminService) validateAgainstSchema(ctx context.Context, isSecret bool, } // If we can't retrieve an active schema, skip validation. - sch, err := s.schr.GetActiveSchema(ctx, s.bindAllocator) + sch, err := s.schr.GetActiveSchema(ctx) if err != nil { logger.Debugf("skipping validation; could not get the active schema: %v", err) return nil diff --git a/backend/controller/admin/admin_test.go b/backend/controller/admin/admin_test.go index 77941e66bc..14856ac013 100644 --- a/backend/controller/admin/admin_test.go +++ b/backend/controller/admin/admin_test.go @@ -13,7 +13,6 @@ import ( "github.com/alecthomas/types/optional" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" - "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/configuration" "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/configuration/providers" @@ -36,7 +35,7 @@ func TestAdminService(t *testing.T) { providers.Inline[configuration.Secrets]{}, }) assert.NoError(t, err) - admin := NewAdminService(cm, sm, &diskSchemaRetriever{}, optional.None[*bind.BindAllocator]()) + admin := NewAdminService(cm, sm, &diskSchemaRetriever{}) assert.NotZero(t, admin) expectedEnvarValue, err := json.MarshalIndent(map[string]string{"bar": "barfoo"}, "", " ") @@ -200,7 +199,7 @@ var testSchema = schema.MustValidate(&schema.Schema{ type mockSchemaRetriever struct { } -func (d *mockSchemaRetriever) GetActiveSchema(ctx context.Context, bindAllocator optional.Option[*bind.BindAllocator]) (*schema.Schema, error) { +func (d *mockSchemaRetriever) GetActiveSchema(ctx context.Context) (*schema.Schema, error) { return testSchema, nil } @@ -218,7 +217,7 @@ func TestAdminValidation(t *testing.T) { providers.Inline[configuration.Secrets]{}, }) assert.NoError(t, err) - admin := NewAdminService(cm, sm, &mockSchemaRetriever{}, optional.None[*bind.BindAllocator]()) + admin := NewAdminService(cm, sm, &mockSchemaRetriever{}) assert.NotZero(t, admin) testSetConfig(t, ctx, admin, "batmobile", "color", "Black", "") diff --git a/backend/controller/admin/local_client.go b/backend/controller/admin/local_client.go index c89f3e214d..a05ef3a239 100644 --- a/backend/controller/admin/local_client.go +++ b/backend/controller/admin/local_client.go @@ -7,8 +7,6 @@ import ( "github.com/alecthomas/types/either" "github.com/alecthomas/types/optional" - "github.com/TBD54566975/ftl/internal/bind" - "github.com/TBD54566975/ftl/internal/buildengine/languageplugin" cf "github.com/TBD54566975/ftl/internal/configuration" "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/errors" @@ -30,15 +28,11 @@ type diskSchemaRetriever struct { } // NewLocalClient creates a admin client that reads and writes from the provided config and secret managers -func NewLocalClient(cm *manager.Manager[cf.Configuration], sm *manager.Manager[cf.Secrets], bindAllocator *bind.BindAllocator) Client { - return &localClient{NewAdminService(cm, sm, &diskSchemaRetriever{}, optional.Some(bindAllocator))} +func NewLocalClient(cm *manager.Manager[cf.Configuration], sm *manager.Manager[cf.Secrets]) Client { + return &localClient{NewAdminService(cm, sm, &diskSchemaRetriever{})} } -func (s *diskSchemaRetriever) GetActiveSchema(ctx context.Context, bAllocator optional.Option[*bind.BindAllocator]) (*schema.Schema, error) { - bindAllocator, ok := bAllocator.Get() - if !ok { - return nil, fmt.Errorf("no bind allocator available") - } +func (s *diskSchemaRetriever) GetActiveSchema(ctx context.Context) (*schema.Schema, error) { path, ok := projectconfig.DefaultConfigPath().Get() if !ok { return nil, fmt.Errorf("no project config path available") @@ -57,23 +51,7 @@ func (s *diskSchemaRetriever) GetActiveSchema(ctx context.Context, bAllocator op for _, m := range modules { go func() { - // Loading a plugin can be expensive. Is there a better way? - plugin, err := languageplugin.New(ctx, bindAllocator, m.Language) - if err != nil { - moduleSchemas <- either.RightOf[*schema.Module](fmt.Errorf("could not load plugin for %s: %w", m.Module, err)) - } - defer plugin.Kill() // nolint:errcheck - - customDefaults, err := plugin.ModuleConfigDefaults(ctx, m.Dir) - if err != nil { - moduleSchemas <- either.RightOf[*schema.Module](fmt.Errorf("could not get module config defaults for %s: %w", m.Module, err)) - } - - config, err := m.FillDefaultsAndValidate(customDefaults) - if err != nil { - moduleSchemas <- either.RightOf[*schema.Module](fmt.Errorf("could not validate module config for %s: %w", m.Module, err)) - } - module, err := schema.ModuleFromProtoFile(config.Abs().Schema()) + module, err := schema.ModuleFromProtoFile(projConfig.SchemaPath(m.Module)) if err != nil { moduleSchemas <- either.RightOf[*schema.Module](fmt.Errorf("could not load module schema: %w", err)) return diff --git a/backend/controller/admin/local_client_test.go b/backend/controller/admin/local_client_test.go index 8b52c9940a..f5050b681f 100644 --- a/backend/controller/admin/local_client_test.go +++ b/backend/controller/admin/local_client_test.go @@ -4,13 +4,11 @@ package admin import ( "context" - "net/url" "testing" "github.com/alecthomas/assert/v2" "github.com/alecthomas/types/optional" - "github.com/TBD54566975/ftl/internal/bind" cf "github.com/TBD54566975/ftl/internal/configuration" "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/configuration/providers" @@ -22,13 +20,8 @@ import ( func getDiskSchema(t testing.TB, ctx context.Context) (*schema.Schema, error) { t.Helper() - - bindURL, err := url.Parse("http://127.0.0.1:8893") - assert.NoError(t, err) - bindAllocator, err := bind.NewBindAllocator(bindURL) - assert.NoError(t, err) dsr := &diskSchemaRetriever{} - return dsr.GetActiveSchema(ctx, optional.Some(bindAllocator)) + return dsr.GetActiveSchema(ctx) } func TestDiskSchemaRetrieverWithBuildArtefact(t *testing.T) { @@ -76,10 +69,10 @@ func TestAdminNoValidationWithNoSchema(t *testing.T) { assert.NoError(t, err) dsr := &diskSchemaRetriever{deployRoot: optional.Some(string(t.TempDir()))} - _, err = dsr.GetActiveSchema(ctx, optional.None[*bind.BindAllocator]()) + _, err = dsr.GetActiveSchema(ctx) assert.Error(t, err) - admin := NewAdminService(cm, sm, dsr, optional.None[*bind.BindAllocator]()) + admin := NewAdminService(cm, sm, dsr) testSetConfig(t, ctx, admin, "batmobile", "color", "Red", "") testSetSecret(t, ctx, admin, "batmobile", "owner", 99, "") } diff --git a/backend/controller/controller.go b/backend/controller/controller.go index 2e36e32cc6..940e8c26f3 100644 --- a/backend/controller/controller.go +++ b/backend/controller/controller.go @@ -57,7 +57,6 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema" frontend "github.com/TBD54566975/ftl/frontend/console" - "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/configuration" cf "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/configuration/providers" @@ -164,7 +163,7 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali cm := cf.ConfigFromContext(ctx) sm := cf.SecretsFromContext(ctx) - admin := admin.NewAdminService(cm, sm, svc.dal, optional.None[*bind.BindAllocator]()) + admin := admin.NewAdminService(cm, sm, svc.dal) console := console.NewService(svc.dal, svc.timeline) ingressHandler := otelhttp.NewHandler(http.Handler(svc), "ftl.ingress") diff --git a/backend/controller/dal/dal.go b/backend/controller/dal/dal.go index 7f0ef5f27a..a6b9e30889 100644 --- a/backend/controller/dal/dal.go +++ b/backend/controller/dal/dal.go @@ -23,7 +23,6 @@ import ( "github.com/TBD54566975/ftl/backend/controller/pubsub" "github.com/TBD54566975/ftl/backend/controller/sql/sqltypes" "github.com/TBD54566975/ftl/backend/libdal" - "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/maps" "github.com/TBD54566975/ftl/internal/model" @@ -584,7 +583,7 @@ func (d *DAL) GetActiveDeployments(ctx context.Context) ([]dalmodel.Deployment, } // GetActiveSchema returns the schema for all active deployments. -func (d *DAL) GetActiveSchema(ctx context.Context, bindAllocator optional.Option[*bind.BindAllocator]) (*schema.Schema, error) { +func (d *DAL) GetActiveSchema(ctx context.Context) (*schema.Schema, error) { deployments, err := d.GetActiveDeployments(ctx) if err != nil { return nil, err diff --git a/frontend/cli/cmd_box.go b/frontend/cli/cmd_box.go index 6f888a93c2..5c392dbfa0 100644 --- a/frontend/cli/cmd_box.go +++ b/frontend/cli/cmd_box.go @@ -132,7 +132,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC } _ = bindAllocator.Next() - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Build.Dirs, bindAllocator, buildengine.BuildEnv(b.Build.BuildEnv), buildengine.Parallelism(b.Build.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig, b.Build.Dirs, bindAllocator, buildengine.BuildEnv(b.Build.BuildEnv), buildengine.Parallelism(b.Build.Parallelism)) if err != nil { return err } @@ -163,7 +163,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC return err } files = append(files, filepath.Join(config.Dir, "ftl.toml")) - files = append(files, config.Schema()) + files = append(files, projConfig.SchemaPath(config.Module)) for _, file := range files { relFile, err := filepath.Rel(config.Dir, file) if err != nil { diff --git a/frontend/cli/cmd_box_run.go b/frontend/cli/cmd_box_run.go index 65b15f25ae..dd340770e6 100644 --- a/frontend/cli/cmd_box_run.go +++ b/frontend/cli/cmd_box_run.go @@ -75,7 +75,7 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er return fmt.Errorf("controller failed to start: %w", err) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), []string{b.Dir}, runnerPortAllocator) + engine, err := buildengine.New(ctx, client, projConfig, []string{b.Dir}, runnerPortAllocator) if err != nil { return fmt.Errorf("failed to create build engine: %w", err) } @@ -85,7 +85,7 @@ func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) er // Manually import the schema for each module to get the dependency graph. err = engine.Each(func(m buildengine.Module) error { logger.Debugf("Loading schema for module %q", m.Config.Module) - mod, err := schema.ModuleFromProtoFile(m.Config.Abs().Schema()) + mod, err := schema.ModuleFromProtoFile(projConfig.SchemaPath(m.Config.Module)) if err != nil { return fmt.Errorf("failed to read schema for module %q: %w", m.Config.Module, err) } diff --git a/frontend/cli/cmd_build.go b/frontend/cli/cmd_build.go index de76c30d5e..aefee0364b 100644 --- a/frontend/cli/cmd_build.go +++ b/frontend/cli/cmd_build.go @@ -31,7 +31,7 @@ func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServic } _ = bindAllocator.Next() - engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, bindAllocator, buildengine.BuildEnv(b.BuildEnv), buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig, b.Dirs, bindAllocator, buildengine.BuildEnv(b.BuildEnv), buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/frontend/cli/cmd_config.go b/frontend/cli/cmd_config.go index b32cde5841..b652f51bb4 100644 --- a/frontend/cli/cmd_config.go +++ b/frontend/cli/cmd_config.go @@ -13,7 +13,6 @@ import ( "github.com/TBD54566975/ftl/backend/controller/admin" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" - "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/configuration" "github.com/TBD54566975/ftl/internal/configuration/manager" "github.com/TBD54566975/ftl/internal/configuration/routers" @@ -87,15 +86,7 @@ func setUpAdminClient(ctx context.Context, config projectconfig.Config) (ctxOut return ctx, client, fmt.Errorf("could not create secrets manager: %w", err) } ctx = manager.ContextWithSecrets(ctx, sm) - - // use the cli endpoint to create the bind allocator, but leave the first port unused as it is meant to be reserved by a controller - bindAllocator, err := bind.NewBindAllocator(cli.Endpoint) - if err != nil { - return ctx, client, fmt.Errorf("could not create bind allocator: %w", err) - } - _ = bindAllocator.Next() - - return ctx, admin.NewLocalClient(cm, sm, bindAllocator), nil + return ctx, admin.NewLocalClient(cm, sm), nil } return ctx, adminServiceClient, nil } diff --git a/frontend/cli/cmd_deploy.go b/frontend/cli/cmd_deploy.go index 3feca9b468..6ae849e564 100644 --- a/frontend/cli/cmd_deploy.go +++ b/frontend/cli/cmd_deploy.go @@ -34,7 +34,7 @@ func (d *deployCmd) Run(ctx context.Context, projConfig projectconfig.Config) er } _ = bindAllocator.Next() - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Build.Dirs, bindAllocator, buildengine.BuildEnv(d.Build.BuildEnv), buildengine.Parallelism(d.Build.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig, d.Build.Dirs, bindAllocator, buildengine.BuildEnv(d.Build.BuildEnv), buildengine.Parallelism(d.Build.Parallelism)) if err != nil { return err } diff --git a/frontend/cli/cmd_dev.go b/frontend/cli/cmd_dev.go index bbc10c3e97..fd054f7f2b 100644 --- a/frontend/cli/cmd_dev.go +++ b/frontend/cli/cmd_dev.go @@ -108,7 +108,7 @@ func (d *devCmd) Run(ctx context.Context, k *kong.Kong, projConfig projectconfig }) } - engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Build.Dirs, bindAllocator, opts...) + engine, err := buildengine.New(ctx, client, projConfig, d.Build.Dirs, bindAllocator, opts...) if err != nil { return err } diff --git a/frontend/cli/cmd_schema_diff.go b/frontend/cli/cmd_schema_diff.go index 60ea4a74c3..4e43d7cf2d 100644 --- a/frontend/cli/cmd_schema_diff.go +++ b/frontend/cli/cmd_schema_diff.go @@ -127,7 +127,7 @@ func localSchema(ctx context.Context, projectConfig projectconfig.Config, bindAl if err != nil { moduleSchemas <- either.RightOf[*schema.Module](err) } - module, err := schema.ModuleFromProtoFile(config.Abs().Schema()) + module, err := schema.ModuleFromProtoFile(projectConfig.SchemaPath(config.Module)) if err != nil { moduleSchemas <- either.RightOf[*schema.Module](err) return diff --git a/internal/buildengine/build.go b/internal/buildengine/build.go index 7c4bdac039..84df269dab 100644 --- a/internal/buildengine/build.go +++ b/internal/buildengine/build.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "time" "github.com/alecthomas/types/result" @@ -14,6 +15,7 @@ import ( "github.com/TBD54566975/ftl/internal/errors" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/moduleconfig" + "github.com/TBD54566975/ftl/internal/projectconfig" "github.com/TBD54566975/ftl/internal/schema" ) @@ -24,16 +26,16 @@ var errInvalidateDependencies = errors.New("dependencies need to be updated") // Plugins must use a lock file to ensure that only one build is running at a time. // // Returns invalidateDependenciesError if the build failed due to a change in dependencies. -func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, bctx languageplugin.BuildContext, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) { +func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectConfig projectconfig.Config, bctx languageplugin.BuildContext, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) { logger := log.FromContext(ctx).Module(bctx.Config.Module).Scope("build") ctx = log.ContextWithLogger(ctx, logger) - stubsRoot := stubsLanguageDir(projectRootDir, bctx.Config.Language) - return handleBuildResult(ctx, bctx.Config, result.From(plugin.Build(ctx, projectRootDir, stubsRoot, bctx, buildEnv, devMode))) + stubsRoot := stubsLanguageDir(projectConfig.Root(), bctx.Config.Language) + return handleBuildResult(ctx, projectConfig, bctx.Config, result.From(plugin.Build(ctx, projectConfig.Root(), stubsRoot, bctx, buildEnv, devMode))) } // handleBuildResult processes the result of a build -func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult result.Result[languageplugin.BuildResult]) (moduleSchema *schema.Module, deploy []string, err error) { +func handleBuildResult(ctx context.Context, projectConfig projectconfig.Config, c moduleconfig.ModuleConfig, eitherResult result.Result[languageplugin.BuildResult]) (moduleSchema *schema.Module, deploy []string, err error) { logger := log.FromContext(ctx) config := c.Abs() @@ -66,7 +68,12 @@ func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherR if err != nil { return nil, nil, fmt.Errorf("failed to marshal schema: %w", err) } - if err := os.WriteFile(config.Schema(), schemaBytes, 0600); err != nil { + schemaPath := projectConfig.SchemaPath(config.Module) + err = os.MkdirAll(filepath.Dir(schemaPath), 0700) + if err != nil { + return nil, nil, fmt.Errorf("failed to create schema directory: %w", err) + } + if err := os.WriteFile(schemaPath, schemaBytes, 0600); err != nil { return nil, nil, fmt.Errorf("failed to write schema: %w", err) } return result.Schema, result.Deploy, nil diff --git a/internal/buildengine/deploy.go b/internal/buildengine/deploy.go index 05a1be3572..7afc1bf274 100644 --- a/internal/buildengine/deploy.go +++ b/internal/buildengine/deploy.go @@ -17,6 +17,7 @@ import ( schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/moduleconfig" + "github.com/TBD54566975/ftl/internal/projectconfig" "github.com/TBD54566975/ftl/internal/sha256" "github.com/TBD54566975/ftl/internal/slices" ) @@ -39,7 +40,7 @@ type DeployClient interface { } // Deploy a module to the FTL controller with the given number of replicas. Optionally wait for the deployment to become ready. -func Deploy(ctx context.Context, module Module, deploy []string, replicas int32, waitForDeployOnline bool, client DeployClient) error { +func Deploy(ctx context.Context, projectConfig projectconfig.Config, module Module, deploy []string, replicas int32, waitForDeployOnline bool, client DeployClient) error { logger := log.FromContext(ctx).Module(module.Config.Module).Scope("deploy") ctx = log.ContextWithLogger(ctx, logger) logger.Infof("Deploying module") @@ -61,9 +62,9 @@ func Deploy(ctx context.Context, module Module, deploy []string, replicas int32, return fmt.Errorf("failed to get artefact diffs: %w", err) } - moduleSchema, err := loadProtoSchema(moduleConfig, replicas) + moduleSchema, err := loadProtoSchema(projectConfig, moduleConfig, replicas) if err != nil { - return fmt.Errorf("failed to load protobuf schema from %q: %w", moduleConfig.Schema(), err) + return err } logger.Debugf("Uploading %d/%d files", len(gadResp.Msg.MissingDigests), len(files)) @@ -135,15 +136,16 @@ func terminateModuleDeployment(ctx context.Context, client DeployClient, module return err } -func loadProtoSchema(config moduleconfig.AbsModuleConfig, replicas int32) (*schemapb.Module, error) { - content, err := os.ReadFile(config.Schema()) +func loadProtoSchema(projectConfig projectconfig.Config, config moduleconfig.AbsModuleConfig, replicas int32) (*schemapb.Module, error) { + schPath := projectConfig.SchemaPath(config.Module) + content, err := os.ReadFile(schPath) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load protobuf schema from %q: %w", schPath, err) } module := &schemapb.Module{} err = proto.Unmarshal(content, module) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to load protobuf schema from %q: %w", schPath, err) } runtime := module.Runtime if runtime == nil { diff --git a/internal/buildengine/engine.go b/internal/buildengine/engine.go index 1bdcff43e0..d390218130 100644 --- a/internal/buildengine/engine.go +++ b/internal/buildengine/engine.go @@ -23,6 +23,7 @@ import ( "github.com/TBD54566975/ftl/internal/buildengine/languageplugin" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/moduleconfig" + "github.com/TBD54566975/ftl/internal/projectconfig" "github.com/TBD54566975/ftl/internal/rpc" "github.com/TBD54566975/ftl/internal/schema" "github.com/TBD54566975/ftl/internal/slices" @@ -177,7 +178,7 @@ type Engine struct { client DeployClient bindAllocator *bind.BindAllocator moduleMetas *xsync.MapOf[string, moduleMeta] - projectRoot string + projectConfig projectconfig.Config moduleDirs []string watcher *watch.Watcher // only watches for module toml changes controllerSchema *xsync.MapOf[string, *schema.Module] @@ -236,11 +237,11 @@ func WithStartTime(startTime time.Time) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client DeployClient, projectRoot string, moduleDirs []string, bindAllocator *bind.BindAllocator, options ...Option) (*Engine, error) { +func New(ctx context.Context, client DeployClient, projectConfig projectconfig.Config, moduleDirs []string, bindAllocator *bind.BindAllocator, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, - projectRoot: projectRoot, + projectConfig: projectConfig, moduleDirs: moduleDirs, moduleMetas: xsync.NewMapOf[string, moduleMeta](), watcher: watch.NewWatcher("ftl.toml"), @@ -261,7 +262,7 @@ func New(ctx context.Context, client DeployClient, projectRoot string, moduleDir ctx, cancel := context.WithCancel(ctx) e.cancel = cancel - err := CleanStubs(ctx, projectRoot) + err := CleanStubs(ctx, projectConfig.Root()) if err != nil { return nil, fmt.Errorf("failed to clean stubs: %w", err) } @@ -450,7 +451,7 @@ func (e *Engine) Deploy(ctx context.Context, replicas int32, waitForDeployOnline return fmt.Errorf("no files found to deploy for %q", moduleName) } e.rawEngineUpdates <- ModuleDeployStarted{Module: moduleName} - err := Deploy(ctx, meta.module, meta.module.Deploy, replicas, waitForDeployOnline, e.client) + err := Deploy(ctx, e.projectConfig, meta.module, meta.module.Deploy, replicas, waitForDeployOnline, e.client) if err != nil { e.rawEngineUpdates <- ModuleDeployFailed{Module: moduleName, Error: err} return err @@ -755,7 +756,7 @@ func (e *Engine) BuildAndDeploy(ctx context.Context, replicas int32, waitForDepl buildGroup.Go(func() error { e.modulesToBuild.Store(module.Config.Module, false) e.rawEngineUpdates <- ModuleDeployStarted{Module: module.Config.Module} - err := Deploy(buildCtx, module, module.Deploy, replicas, waitForDeployOnline, e.client) + err := Deploy(buildCtx, e.projectConfig, module, module.Deploy, replicas, waitForDeployOnline, e.client) if err != nil { e.rawEngineUpdates <- ModuleDeployFailed{Module: module.Config.Module, Error: err} return err @@ -850,7 +851,7 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, metasMap[name] = meta return true }) - err = GenerateStubs(ctx, e.projectRoot, maps.Values(knownSchemas), metasMap) + err = GenerateStubs(ctx, e.projectConfig.Root(), maps.Values(knownSchemas), metasMap) if err != nil { return err } @@ -889,7 +890,7 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, } // Sync references to stubs if needed by the runtime - err = SyncStubReferences(ctx, e.projectRoot, moduleNames, metasMap) + err = SyncStubReferences(ctx, e.projectConfig.Root(), moduleNames, metasMap) if err != nil { return err } @@ -966,7 +967,7 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ sch := &schema.Schema{Modules: maps.Values(builtModules)} - moduleSchema, deploy, err := build(ctx, meta.plugin, e.projectRoot, languageplugin.BuildContext{ + moduleSchema, deploy, err := build(ctx, meta.plugin, e.projectConfig, languageplugin.BuildContext{ Config: meta.module.Config, Schema: sch, Dependencies: meta.module.Dependencies(Raw), @@ -1074,7 +1075,7 @@ func (e *Engine) watchForPluginEvents(originalCtx context.Context) { e.rawEngineUpdates <- ModuleBuildStarted{Config: meta.module.Config, IsAutoRebuild: true} case languageplugin.AutoRebuildEndedEvent: - _, deploy, err := handleBuildResult(ctx, meta.module.Config, event.Result) + _, deploy, err := handleBuildResult(ctx, e.projectConfig, meta.module.Config, event.Result) if err != nil { e.rawEngineUpdates <- ModuleBuildFailed{Config: meta.module.Config, IsAutoRebuild: true, Error: err} if errors.Is(err, errInvalidateDependencies) { @@ -1087,7 +1088,7 @@ func (e *Engine) watchForPluginEvents(originalCtx context.Context) { e.rawEngineUpdates <- ModuleBuildSuccess{Config: meta.module.Config, IsAutoRebuild: true} e.rawEngineUpdates <- ModuleDeployStarted{Module: event.Module} - if err := Deploy(ctx, meta.module, deploy, 1, true, e.client); err != nil { + if err := Deploy(ctx, e.projectConfig, meta.module, deploy, 1, true, e.client); err != nil { e.rawEngineUpdates <- ModuleDeployFailed{Module: event.Module, Error: err} continue } diff --git a/internal/buildengine/engine_test.go b/internal/buildengine/engine_test.go index 3c3e82bf9c..d4bdf320f0 100644 --- a/internal/buildengine/engine_test.go +++ b/internal/buildengine/engine_test.go @@ -3,6 +3,7 @@ package buildengine_test import ( "context" "net/url" + "path/filepath" "testing" "github.com/alecthomas/assert/v2" @@ -10,6 +11,7 @@ import ( "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/buildengine" "github.com/TBD54566975/ftl/internal/log" + "github.com/TBD54566975/ftl/internal/projectconfig" "github.com/TBD54566975/ftl/internal/schema" ) @@ -21,7 +23,11 @@ func TestGraph(t *testing.T) { bindAllocator, err := bind.NewBindAllocator(bindURL) assert.NoError(t, err) - engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/alpha", "testdata/other", "testdata/another"}, bindAllocator) + projConfig := projectconfig.Config{ + Path: filepath.Join(t.TempDir(), "ftl-project.toml"), + Name: "test", + } + engine, err := buildengine.New(ctx, nil, projConfig, []string{"testdata/alpha", "testdata/other", "testdata/another"}, bindAllocator) assert.NoError(t, err) defer engine.Close() diff --git a/internal/buildengine/languageplugin/java_plugin.go b/internal/buildengine/languageplugin/java_plugin.go index 65f120da7c..1a8481912b 100644 --- a/internal/buildengine/languageplugin/java_plugin.go +++ b/internal/buildengine/languageplugin/java_plugin.go @@ -281,7 +281,7 @@ func buildJava(ctx context.Context, projectRoot, stubsRoot string, bctx BuildCon return result, nil } - moduleSchema, err := schema.ModuleFromProtoFile(config.Schema()) + moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(config.DeployDir, "schema.pb")) if err != nil { return BuildResult{}, fmt.Errorf("failed to read schema for module: %w", err) } diff --git a/internal/buildengine/languageplugin/rust_plugin.go b/internal/buildengine/languageplugin/rust_plugin.go index f365386e0f..c0eea95208 100644 --- a/internal/buildengine/languageplugin/rust_plugin.go +++ b/internal/buildengine/languageplugin/rust_plugin.go @@ -3,6 +3,7 @@ package languageplugin import ( "context" "fmt" + "path/filepath" "github.com/alecthomas/kong" "github.com/alecthomas/types/optional" @@ -69,7 +70,7 @@ func buildRust(ctx context.Context, projectRoot, stubsRoot string, bctx BuildCon return result, nil } - moduleSchema, err := schema.ModuleFromProtoFile(config.Schema()) + moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(config.DeployDir, "schema.pb")) if err != nil { return BuildResult{}, fmt.Errorf("failed to read schema for module: %w", err) } diff --git a/internal/moduleconfig/moduleconfig.go b/internal/moduleconfig/moduleconfig.go index e5e0e1ad23..e82c6af86d 100644 --- a/internal/moduleconfig/moduleconfig.go +++ b/internal/moduleconfig/moduleconfig.go @@ -191,11 +191,3 @@ func isBeneath(moduleDir, path string) bool { resolved := filepath.Clean(filepath.Join(moduleDir, path)) return strings.HasPrefix(resolved, strings.TrimSuffix(moduleDir, "/")+"/") } - -func (c ModuleConfig) Schema() string { - return "schema.pb" -} - -func (c AbsModuleConfig) Schema() string { - return filepath.Join(c.DeployDir, "schema.pb") -} diff --git a/internal/projectconfig/projectconfig.go b/internal/projectconfig/projectconfig.go index def134678b..0d7134d5bf 100644 --- a/internal/projectconfig/projectconfig.go +++ b/internal/projectconfig/projectconfig.go @@ -199,3 +199,8 @@ func Save(config Config) error { } return os.Rename(w.Name(), config.Path) } + +// SchemaPath returns the path to the schema file for the given module. +func (c Config) SchemaPath(module string) string { + return filepath.Join(c.Root(), ".ftl", "schemas", module+".pb") +}