From 5a303ee22bef3c4c1cb07ebbe3e7ac92b718fc82 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 8 Jul 2024 10:31:13 -0700 Subject: [PATCH] fix: use shared .ftl folder for stubs (#1843) (#2005) Fixes #1662 Fixes #1827 --- .gitignore | 6 + Justfile | 2 +- buildengine/build.go | 8 +- buildengine/build_go.go | 4 +- buildengine/build_go_test.go | 249 +---------------- buildengine/build_test.go | 36 ++- buildengine/deploy_test.go | 11 +- buildengine/discover_test.go | 14 +- buildengine/engine.go | 50 +++- buildengine/engine_test.go | 4 +- buildengine/stubs.go | 39 +++ buildengine/stubs_test.go | 250 ++++++++++++++++++ cmd/ftl/cmd_box.go | 2 +- cmd/ftl/cmd_box_run.go | 5 +- cmd/ftl/cmd_build.go | 2 +- cmd/ftl/cmd_deploy.go | 5 +- cmd/ftl/cmd_dev.go | 2 +- cmd/ftl/cmd_init.go | 4 +- cmd/ftl/cmd_new.go | 10 +- common/moduleconfig/moduleconfig.go | 2 +- extensions/vscode/package-lock.json | 4 +- .../go/main/go.mod.tmpl | 2 +- .../go/main/main.go.tmpl} | 0 .../compile/build-template/go.work.tmpl | 6 +- go-runtime/compile/build.go | 195 +++++++++----- go-runtime/compile/devel.go | 5 + .../external_module.go.tmpl | 32 +-- .../modules/{{ .Module.Name }}}/go.mod.tmpl | 4 +- .../compile/external-module-template/go.mod | 3 - .../external-module-template/go.work.tmpl | 6 - .../compile/main-work-template/go.work.tmpl | 8 + go-runtime/compile/release.go | 11 + go-runtime/compile/schema_test.go | 8 +- .../{{ .Name | camel | lower }}/go.mod.tmpl | 4 +- integration/harness.go | 23 +- 35 files changed, 623 insertions(+), 393 deletions(-) create mode 100644 buildengine/stubs.go create mode 100644 buildengine/stubs_test.go rename go-runtime/compile/build-template/{_ftl.tmpl => .ftl.tmpl}/go/main/go.mod.tmpl (94%) rename go-runtime/compile/build-template/{_ftl.tmpl/go/main/main.go => .ftl.tmpl/go/main/main.go.tmpl} (100%) rename go-runtime/compile/external-module-template/{_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }} => .ftl/go/modules/{{ .Module.Name }}}/external_module.go.tmpl (66%) rename go-runtime/compile/external-module-template/{_ftl/go/modules => .ftl/go/modules/{{ .Module.Name }}}/go.mod.tmpl (80%) delete mode 100644 go-runtime/compile/external-module-template/go.mod delete mode 100644 go-runtime/compile/external-module-template/go.work.tmpl create mode 100644 go-runtime/compile/main-work-template/go.work.tmpl diff --git a/.gitignore b/.gitignore index e54ce37176..31dcf44481 100644 --- a/.gitignore +++ b/.gitignore @@ -20,9 +20,15 @@ examples/**/go.work examples/**/go.work.sum testdata/**/go.work testdata/**/go.work.sum + +# Leaving old _ftl for now to avoid old stuff getting checked in **/testdata/**/_ftl **/examples/**/_ftl **/examples/**/types.ftl.go +**/testdata/**/.ftl +**/examples/**/.ftl +/.ftl + buildengine/.gitignore go.work* junit*.xml diff --git a/Justfile b/Justfile index 0f6d1eb3f3..50af62da26 100644 --- a/Justfile +++ b/Justfile @@ -9,7 +9,7 @@ KT_RUNTIME_RUNNER_TEMPLATE_OUT := "build/template/ftl/jars/ftl-runtime.jar" RUNNER_TEMPLATE_ZIP := "backend/controller/scaling/localscaling/template.zip" TIMESTAMP := `date +%s` SCHEMA_OUT := "backend/protos/xyz/block/ftl/v1/schema/schema.proto" -ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" +ZIP_DIRS := "go-runtime/compile/build-template go-runtime/compile/external-module-template go-runtime/compile/main-work-template common-runtime/scaffolding go-runtime/scaffolding kotlin-runtime/scaffolding kotlin-runtime/external-module-template" FRONTEND_OUT := "frontend/dist/index.html" EXTENSION_OUT := "extensions/vscode/dist/extension.js" PROTOS_IN := "backend/protos/xyz/block/ftl/v1/schema/schema.proto backend/protos/xyz/block/ftl/v1/console/console.proto backend/protos/xyz/block/ftl/v1/ftl.proto backend/protos/xyz/block/ftl/v1/schema/runtime.proto" diff --git a/buildengine/build.go b/buildengine/build.go index 2987223997..3a50e0522e 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -22,11 +22,11 @@ const BuildLockTimeout = time.Minute // Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func Build(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { - return buildModule(ctx, sch, module, filesTransaction) +func Build(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { + return buildModule(ctx, projectRootDir, sch, module, filesTransaction) } -func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { +func buildModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { release, err := flock.Acquire(ctx, filepath.Join(module.Config.Dir, ".ftl.lock"), BuildLockTimeout) if err != nil { return err @@ -46,7 +46,7 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTr switch module.Config.Language { case "go": - err = buildGoModule(ctx, sch, module, filesTransaction) + err = buildGoModule(ctx, projectRootDir, sch, module, filesTransaction) case "kotlin": err = buildKotlinModule(ctx, sch, module) case "rust": diff --git a/buildengine/build_go.go b/buildengine/build_go.go index ee8fb1c98b..8eba01a747 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -8,8 +8,8 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGoModule(ctx context.Context, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { - if err := compile.Build(ctx, module.Config.Dir, sch, transaction); err != nil { +func buildGoModule(ctx context.Context, projectRootDir string, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { + if err := compile.Build(ctx, projectRootDir, module.Config.Dir, sch, transaction); err != nil { return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", module.Config.Module, err)} } return nil diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index 498e26e43b..cdfd9c8210 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -11,181 +11,6 @@ import ( "github.com/TBD54566975/ftl/backend/schema" ) -func TestGenerateGoModule(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - sch := &schema.Schema{ - Modules: []*schema.Module{ - schema.Builtins(), - {Name: "other", Decls: []schema.Decl{ - &schema.Enum{ - Comments: []string{"This is an enum.", "", "It has 3 variants."}, - Name: "Color", - Export: true, - Type: &schema.String{}, - Variants: []*schema.EnumVariant{ - {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, - {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, - {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, - }, - }, - &schema.Enum{ - Name: "ColorInt", - Export: true, - Type: &schema.Int{}, - Variants: []*schema.EnumVariant{ - {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, - {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, - {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, - }, - }, - &schema.Enum{ - Comments: []string{"This is type enum."}, - Name: "TypeEnum", - Export: true, - Variants: []*schema.EnumVariant{ - {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, - {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, - }, - }, - &schema.Data{Name: "EchoRequest", Export: true}, - &schema.Data{ - Comments: []string{"This is an echo data response."}, - Name: "EchoResponse", Export: true}, - &schema.Verb{ - Name: "echo", - Export: true, - Request: &schema.Ref{Name: "EchoRequest"}, - Response: &schema.Ref{Name: "EchoResponse"}, - }, - &schema.Data{Name: "SinkReq", Export: true}, - &schema.Verb{ - Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, - Name: "sink", - Export: true, - Request: &schema.Ref{Name: "SinkReq"}, - Response: &schema.Unit{}, - }, - &schema.Data{Name: "SourceResp", Export: true}, - &schema.Verb{ - Name: "source", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Ref{Name: "SourceResp"}, - }, - &schema.Verb{ - Name: "nothing", - Export: true, - Request: &schema.Unit{}, - Response: &schema.Unit{}, - }, - }}, - {Name: "test"}, - }, - } - expected := `// Code generated by FTL. DO NOT EDIT. - -package other - -import ( - "context" - - "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" -) - -var _ = context.Background - -// This is an enum. -// -// It has 3 variants. -// -//ftl:enum -type Color string -const ( - Red Color = "Red" - Blue Color = "Blue" - Green Color = "Green" -) - -//ftl:enum -type ColorInt int -const ( - RedInt ColorInt = 0 - BlueInt ColorInt = 1 - GreenInt ColorInt = 2 -) - -// This is type enum. -// -//ftl:enum -type TypeEnum interface { typeEnum() } - -type A int - -func (A) typeEnum() {} - -type B string - -func (B) typeEnum() {} - -type EchoRequest struct { -} - -// This is an echo data response. -// -type EchoResponse struct { -} - -//ftl:verb -func Echo(context.Context, EchoRequest) (EchoResponse, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} - -type SinkReq struct { -} - -// This is a sink verb. -// -// Here is another line for this comment! -// -//ftl:verb -func Sink(context.Context, SinkReq) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") -} - -type SourceResp struct { -} - -//ftl:verb -func Source(context.Context) (SourceResp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") -} - -//ftl:verb -func Nothing(context.Context) error { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") -} - -func init() { - reflection.Register( - reflection.SumType[TypeEnum]( - *new(A), - *new(B), - ), - ) -} -` - bctx := buildContext{ - moduleDir: "testdata/another", - buildDir: "_ftl", - sch: sch, - } - testBuild(t, bctx, "", []assertion{ - assertGeneratedModule("go/modules/other/external_module.go", expected), - }) -} - func TestGoBuildClearsBuildDir(t *testing.T) { if testing.Short() { t.SkipNow() @@ -198,82 +23,22 @@ func TestGoBuildClearsBuildDir(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/another", - buildDir: "_ftl", + buildDir: ".ftl", sch: sch, } testBuildClearsBuildDir(t, bctx) } -func TestMetadataImportsExcluded(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - sch := &schema.Schema{ - Modules: []*schema.Module{ - schema.Builtins(), - {Name: "test", Decls: []schema.Decl{ - &schema.Data{ - Comments: []string{"Request data type."}, - Name: "Req", Export: true}, - &schema.Data{Name: "Resp", Export: true}, - &schema.Verb{ - Comments: []string{"This is a verb."}, - Name: "call", - Export: true, - Request: &schema.Ref{Name: "Req"}, - Response: &schema.Ref{Name: "Resp"}, - Metadata: []schema.Metadata{ - &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, - }, - }, - }}, - }, - } - expected := `// Code generated by FTL. DO NOT EDIT. - -package test - -import ( - "context" -) - -var _ = context.Background - -// Request data type. -// -type Req struct { -} - -type Resp struct { -} - -// This is a verb. -// -//ftl:verb -func Call(context.Context, Req) (Resp, error) { - panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") -} -` - bctx := buildContext{ - moduleDir: "testdata/another", - buildDir: "_ftl", - sch: sch, - } - testBuild(t, bctx, "", []assertion{ - assertGeneratedModule("go/modules/test/external_module.go", expected), - }) -} - func TestExternalType(t *testing.T) { if testing.Short() { t.SkipNow() } bctx := buildContext{ moduleDir: "testdata/external", - buildDir: "_ftl", + buildDir: ".ftl", sch: &schema.Schema{}, } - testBuild(t, bctx, "unsupported external type", []assertion{ + testBuild(t, bctx, "", "unsupported external type", []assertion{ assertBuildProtoErrors( "unsupported external type \"time.Month\"", "unsupported type \"time.Month\" for field \"Month\"", @@ -302,10 +67,10 @@ func TestGoModVersion(t *testing.T) { } bctx := buildContext{ moduleDir: "testdata/highgoversion", - buildDir: "_ftl", + buildDir: ".ftl", sch: sch, } - testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), []assertion{}) + testBuild(t, bctx, fmt.Sprintf("go version %q is not recent enough for this module, needs minimum version \"9000.1.1\"", runtime.Version()[2:]), "", []assertion{}) } func TestGeneratedTypeRegistry(t *testing.T) { @@ -345,10 +110,10 @@ func TestGeneratedTypeRegistry(t *testing.T) { assert.NoError(t, err) bctx := buildContext{ moduleDir: "testdata/other", - buildDir: "_ftl", + buildDir: ".ftl", sch: sch, } - testBuild(t, bctx, "", []assertion{ + testBuild(t, bctx, "", "", []assertion{ assertGeneratedMain(string(expected)), }) } diff --git a/buildengine/build_test.go b/buildengine/build_test.go index 83a05b6c09..e29e244ea2 100644 --- a/buildengine/build_test.go +++ b/buildengine/build_test.go @@ -38,6 +38,7 @@ func (t *mockModifyFilesTransaction) End() error { func testBuild( t *testing.T, bctx buildContext, + expectedGeneratStubsErrMsg string, // emptystr if no error expected expectedBuildErrMsg string, // emptystr if no error expected assertions []assertion, ) { @@ -47,12 +48,31 @@ func testBuild( assert.NoError(t, err, "Error getting absolute path for module directory") module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) - if len(expectedBuildErrMsg) > 0 { + + projectRootDir := t.TempDir() + + configs := []moduleconfig.ModuleConfig{} + if bctx.moduleDir != "" { + config, err := moduleconfig.LoadModuleConfig(bctx.moduleDir) + assert.NoError(t, err, "Error loading project config") + configs = append(configs, config) + } + + // generate stubs to create the shared modules directory + err = GenerateStubs(ctx, projectRootDir, bctx.sch.Modules, configs) + if len(expectedGeneratStubsErrMsg) > 0 { assert.Error(t, err) - assert.Contains(t, err.Error(), expectedBuildErrMsg) + assert.Contains(t, err.Error(), expectedGeneratStubsErrMsg) } else { assert.NoError(t, err) + + err = Build(ctx, projectRootDir, bctx.sch, module, &mockModifyFilesTransaction{}) + if len(expectedBuildErrMsg) > 0 { + assert.Error(t, err) + assert.Contains(t, err.Error(), expectedBuildErrMsg) + } else { + assert.NoError(t, err) + } } for _, a := range assertions { @@ -70,10 +90,16 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { abs, err := filepath.Abs(bctx.moduleDir) assert.NoError(t, err, "Error getting absolute path for module directory") + projectRoot := t.TempDir() + + // generate stubs to create the shared modules directory + err = GenerateStubs(ctx, projectRoot, bctx.sch.Modules, []moduleconfig.ModuleConfig{{Dir: bctx.moduleDir}}) + assert.NoError(t, err) + // build to generate the build directory module, err := LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // create a temporary file in the build directory @@ -85,7 +111,7 @@ func testBuildClearsBuildDir(t *testing.T, bctx buildContext) { // build to clear the old build directory module, err = LoadModule(abs) assert.NoError(t, err) - err = Build(ctx, bctx.sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, projectRoot, bctx.sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) // ensure the temporary file was removed diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index 029f2d33af..3550668d26 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -10,6 +10,7 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/sha256" ) @@ -71,11 +72,17 @@ func TestDeploy(t *testing.T) { module, err := LoadModule(modulePath) assert.NoError(t, err) + projectRootDir := t.TempDir() + + // generate stubs to create the shared modules directory + err = GenerateStubs(ctx, projectRootDir, sch.Modules, []moduleconfig.ModuleConfig{module.Config}) + assert.NoError(t, err) + // Build first to make sure the files are there. - err = Build(ctx, sch, module, &mockModifyFilesTransaction{}) + err = Build(ctx, projectRootDir, sch, module, &mockModifyFilesTransaction{}) assert.NoError(t, err) - sum, err := sha256.SumFile(modulePath + "/_ftl/main") + sum, err := sha256.SumFile(modulePath + "/.ftl/main") assert.NoError(t, err) client := &mockDeployClient{ diff --git a/buildengine/discover_test.go b/buildengine/discover_test.go index b63d7d3416..0aec6ccc97 100644 --- a/buildengine/discover_test.go +++ b/buildengine/discover_test.go @@ -22,7 +22,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "alpha", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -35,7 +35,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "another", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -48,7 +48,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle1", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -61,7 +61,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "depcycle2", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum"}, @@ -100,7 +100,7 @@ func TestDiscoverModules(t *testing.T) { Deploy: []string{ "main", }, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{ @@ -140,7 +140,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "highgoversion", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, @@ -153,7 +153,7 @@ func TestDiscoverModules(t *testing.T) { Realm: "home", Module: "other", Deploy: []string{"main"}, - DeployDir: "_ftl", + DeployDir: ".ftl", Schema: "schema.pb", Errors: "errors.pb", Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, diff --git a/buildengine/engine.go b/buildengine/engine.go index b9a7b4215b..af9c5d14be 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -21,6 +21,7 @@ import ( ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/rpc" "github.com/TBD54566975/ftl/internal/slices" @@ -65,6 +66,7 @@ type Listener interface { type Engine struct { client ftlv1connect.ControllerServiceClient moduleMetas *xsync.MapOf[string, moduleMeta] + projectRoot string moduleDirs []string watcher *Watcher controllerSchema *xsync.MapOf[string, *schema.Module] @@ -97,10 +99,11 @@ func WithListener(listener Listener) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, options ...Option) (*Engine, error) { +func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, projectRoot string, moduleDirs []string, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, + projectRoot: projectRoot, moduleDirs: moduleDirs, moduleMetas: xsync.NewMapOf[string, moduleMeta](), watcher: NewWatcher(), @@ -116,6 +119,11 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, modul ctx, cancel := context.WithCancel(ctx) e.cancel = cancel + err := CleanStubs(ctx, projectRoot) + if err != nil { + return nil, fmt.Errorf("failed to clean stubs: %w", err) + } + modules, err := DiscoverModules(ctx, moduleDirs) if err != nil { return nil, fmt.Errorf("could not find modules: %w", err) @@ -566,6 +574,21 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, } errCh := make(chan error, 1024) for _, group := range topology { + groupSchemas := map[string]*schema.Module{} + metas, err := e.gatherGroupSchemas(builtModules, group, groupSchemas) + if err != nil { + return err + } + + moduleConfigs := make([]moduleconfig.ModuleConfig, len(metas)) + for i, meta := range metas { + moduleConfigs[i] = meta.module.Config + } + err = GenerateStubs(ctx, e.projectRoot, maps.Values(groupSchemas), moduleConfigs) + if err != nil { + return err + } + // Collect schemas to be inserted into "built" map for subsequent groups. schemas := make(chan *schema.Module, len(group)) @@ -664,7 +687,7 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ if e.listener != nil { e.listener.OnBuildStarted(meta.module) } - err := Build(ctx, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) + err := Build(ctx, e.projectRoot, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) if err != nil { return err } @@ -677,6 +700,29 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[ return nil } +// Construct a combined schema for a group of modules and their transitive dependencies. +func (e *Engine) gatherGroupSchemas( + moduleSchemas map[string]*schema.Module, + group []string, + out map[string]*schema.Module, +) ([]moduleMeta, error) { + var metas []moduleMeta + for _, module := range group { + if module == "builtin" { + continue // Skip the builtin module + } + + meta, ok := e.moduleMetas.Load(module) + if ok { + metas = append(metas, meta) + if err := e.gatherSchemas(moduleSchemas, meta.module, out); err != nil { + return nil, err + } + } + } + return metas, nil +} + // Construct a combined schema for a module and its transitive dependencies. func (e *Engine) gatherSchemas( moduleSchemas map[string]*schema.Module, diff --git a/buildengine/engine_test.go b/buildengine/engine_test.go index 73a5211349..a44ab8768a 100644 --- a/buildengine/engine_test.go +++ b/buildengine/engine_test.go @@ -16,7 +16,7 @@ func TestEngine(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/alpha", "testdata/other", "testdata/another"}) + engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/alpha", "testdata/other", "testdata/another"}) assert.NoError(t, err) defer engine.Close() @@ -64,7 +64,7 @@ func TestCycleDetection(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/depcycle1", "testdata/depcycle2"}) + engine, err := buildengine.New(ctx, nil, t.TempDir(), []string{"testdata/depcycle1", "testdata/depcycle2"}) assert.NoError(t, err) defer engine.Close() diff --git a/buildengine/stubs.go b/buildengine/stubs.go new file mode 100644 index 0000000000..38731b7bc9 --- /dev/null +++ b/buildengine/stubs.go @@ -0,0 +1,39 @@ +package buildengine + +import ( + "context" + "fmt" + + "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" + "github.com/TBD54566975/ftl/go-runtime/compile" +) + +// GenerateStubs generates stubs for the given modules. +// +// Currently, only Go stubs are supported. Kotlin and other language stubs can be added in the future. +func GenerateStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { + return generateGoStubs(ctx, projectRoot, modules, moduleConfigs) +} + +// CleanStubs removes all generated stubs. +func CleanStubs(ctx context.Context, projectRoot string) error { + return cleanGoStubs(ctx, projectRoot) +} + +func generateGoStubs(ctx context.Context, projectRoot string, modules []*schema.Module, moduleConfigs []moduleconfig.ModuleConfig) error { + sch := &schema.Schema{Modules: modules} + err := compile.GenerateStubsForModules(ctx, projectRoot, moduleConfigs, sch) + if err != nil { + return fmt.Errorf("failed to generate go stubs: %w", err) + } + return nil +} + +func cleanGoStubs(ctx context.Context, projectRoot string) error { + err := compile.CleanStubs(ctx, projectRoot) + if err != nil { + return fmt.Errorf("failed to clean go stubs: %w", err) + } + return nil +} diff --git a/buildengine/stubs_test.go b/buildengine/stubs_test.go new file mode 100644 index 0000000000..c4c1efe5b5 --- /dev/null +++ b/buildengine/stubs_test.go @@ -0,0 +1,250 @@ +package buildengine + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/TBD54566975/ftl/backend/schema" + "github.com/TBD54566975/ftl/common/moduleconfig" + "github.com/TBD54566975/ftl/internal/log" + "github.com/alecthomas/assert/v2" +) + +func TestGenerateGoStubs(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + modules := []*schema.Module{ + schema.Builtins(), + {Name: "other", Decls: []schema.Decl{ + &schema.Enum{ + Comments: []string{"This is an enum.", "", "It has 3 variants."}, + Name: "Color", + Export: true, + Type: &schema.String{}, + Variants: []*schema.EnumVariant{ + {Name: "Red", Value: &schema.StringValue{Value: "Red"}}, + {Name: "Blue", Value: &schema.StringValue{Value: "Blue"}}, + {Name: "Green", Value: &schema.StringValue{Value: "Green"}}, + }, + }, + &schema.Enum{ + Name: "ColorInt", + Export: true, + Type: &schema.Int{}, + Variants: []*schema.EnumVariant{ + {Name: "RedInt", Value: &schema.IntValue{Value: 0}}, + {Name: "BlueInt", Value: &schema.IntValue{Value: 1}}, + {Name: "GreenInt", Value: &schema.IntValue{Value: 2}}, + }, + }, + &schema.Enum{ + Comments: []string{"This is type enum."}, + Name: "TypeEnum", + Export: true, + Variants: []*schema.EnumVariant{ + {Name: "A", Value: &schema.TypeValue{Value: &schema.Int{}}}, + {Name: "B", Value: &schema.TypeValue{Value: &schema.String{}}}, + }, + }, + &schema.Data{Name: "EchoRequest", Export: true}, + &schema.Data{ + Comments: []string{"This is an echo data response."}, + Name: "EchoResponse", Export: true}, + &schema.Verb{ + Name: "echo", + Export: true, + Request: &schema.Ref{Name: "EchoRequest"}, + Response: &schema.Ref{Name: "EchoResponse"}, + }, + &schema.Data{Name: "SinkReq", Export: true}, + &schema.Verb{ + Comments: []string{"This is a sink verb.", "", "Here is another line for this comment!"}, + Name: "sink", + Export: true, + Request: &schema.Ref{Name: "SinkReq"}, + Response: &schema.Unit{}, + }, + &schema.Data{Name: "SourceResp", Export: true}, + &schema.Verb{ + Name: "source", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Ref{Name: "SourceResp"}, + }, + &schema.Verb{ + Name: "nothing", + Export: true, + Request: &schema.Unit{}, + Response: &schema.Unit{}, + }, + }}, + {Name: "test"}, + } + + expected := `// Code generated by FTL. DO NOT EDIT. + +package other + +import ( + "context" + + "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" +) + +var _ = context.Background + +// This is an enum. +// +// It has 3 variants. +// +//ftl:enum +type Color string +const ( + Red Color = "Red" + Blue Color = "Blue" + Green Color = "Green" +) + +//ftl:enum +type ColorInt int +const ( + RedInt ColorInt = 0 + BlueInt ColorInt = 1 + GreenInt ColorInt = 2 +) + +// This is type enum. +// +//ftl:enum +type TypeEnum interface { typeEnum() } + +type A int + +func (A) typeEnum() {} + +type B string + +func (B) typeEnum() {} + +type EchoRequest struct { +} + +// This is an echo data response. +// +type EchoResponse struct { +} + +//ftl:verb +func Echo(context.Context, EchoRequest) (EchoResponse, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} + +type SinkReq struct { +} + +// This is a sink verb. +// +// Here is another line for this comment! +// +//ftl:verb +func Sink(context.Context, SinkReq) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") +} + +type SourceResp struct { +} + +//ftl:verb +func Source(context.Context) (SourceResp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") +} + +//ftl:verb +func Nothing(context.Context) error { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") +} + +func init() { + reflection.Register( + reflection.SumType[TypeEnum]( + *new(A), + *new(B), + ), + ) +} +` + + ctx := log.ContextWithNewDefaultLogger(context.Background()) + projectRoot := t.TempDir() + err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) + assert.NoError(t, err) + + generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/other/external_module.go") + fileContent, err := os.ReadFile(generatedPath) + assert.NoError(t, err) + assert.Equal(t, expected, string(fileContent)) +} + +func TestMetadataImportsExcluded(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + modules := []*schema.Module{ + schema.Builtins(), + {Name: "test", Decls: []schema.Decl{ + &schema.Data{ + Comments: []string{"Request data type."}, + Name: "Req", Export: true}, + &schema.Data{Name: "Resp", Export: true}, + &schema.Verb{ + Comments: []string{"This is a verb."}, + Name: "call", + Export: true, + Request: &schema.Ref{Name: "Req"}, + Response: &schema.Ref{Name: "Resp"}, + Metadata: []schema.Metadata{ + &schema.MetadataCalls{Calls: []*schema.Ref{{Name: "verb", Module: "other"}}}, + }, + }, + }}, + } + + expected := `// Code generated by FTL. DO NOT EDIT. + +package test + +import ( + "context" +) + +var _ = context.Background + +// Request data type. +// +type Req struct { +} + +type Resp struct { +} + +// This is a verb. +// +//ftl:verb +func Call(context.Context, Req) (Resp, error) { + panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") +} +` + ctx := log.ContextWithNewDefaultLogger(context.Background()) + projectRoot := t.TempDir() + err := GenerateStubs(ctx, projectRoot, modules, []moduleconfig.ModuleConfig{}) + assert.NoError(t, err) + + generatedPath := filepath.Join(projectRoot, ".ftl/go/modules/test/external_module.go") + fileContent, err := os.ReadFile(generatedPath) + assert.NoError(t, err) + assert.Equal(t, expected, string(fileContent)) +} diff --git a/cmd/ftl/cmd_box.go b/cmd/ftl/cmd_box.go index d09214c887..887eefb763 100644 --- a/cmd/ftl/cmd_box.go +++ b/cmd/ftl/cmd_box.go @@ -123,7 +123,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_box_run.go b/cmd/ftl/cmd_box_run.go index 0f9cb08b52..0565edd540 100644 --- a/cmd/ftl/cmd_box_run.go +++ b/cmd/ftl/cmd_box_run.go @@ -17,6 +17,7 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/buildengine" + "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/bind" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/model" @@ -33,7 +34,7 @@ type boxRunCmd struct { ControllerTimeout time.Duration `help:"Timeout for Controller start." default:"30s"` } -func (b *boxRunCmd) Run(ctx context.Context) error { +func (b *boxRunCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { conn, err := databasetesting.CreateForDevel(ctx, b.DSN, b.Recreate) if err != nil { return fmt.Errorf("failed to create database: %w", err) @@ -74,7 +75,7 @@ func (b *boxRunCmd) Run(ctx context.Context) error { return fmt.Errorf("controller failed to start: %w", err) } - engine, err := buildengine.New(ctx, client, []string{b.Dir}) + engine, err := buildengine.New(ctx, client, projConfig.Root(), []string{b.Dir}) if err != nil { return fmt.Errorf("failed to create build engine: %w", err) } diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index 35e826e740..a3f13869b2 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -22,7 +22,7 @@ func (b *buildCmd) Run(ctx context.Context, client ftlv1connect.ControllerServic if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index 0ecd47a6ed..a50c7f25fc 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -5,6 +5,7 @@ import ( "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" "github.com/TBD54566975/ftl/buildengine" + "github.com/TBD54566975/ftl/common/projectconfig" "github.com/TBD54566975/ftl/internal/rpc" ) @@ -15,9 +16,9 @@ type deployCmd struct { NoWait bool `help:"Do not wait for deployment to complete." default:"false"` } -func (d *deployCmd) Run(ctx context.Context) error { +func (d *deployCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go index 2741faecfe..e596d84fa3 100644 --- a/cmd/ftl/cmd_dev.go +++ b/cmd/ftl/cmd_dev.go @@ -89,7 +89,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error }) } - engine, err := buildengine.New(ctx, client, d.Dirs, opts...) + engine, err := buildengine.New(ctx, client, projConfig.Root(), d.Dirs, opts...) if err != nil { return err } diff --git a/cmd/ftl/cmd_init.go b/cmd/ftl/cmd_init.go index b394639797..b2cafb64e3 100644 --- a/cmd/ftl/cmd_init.go +++ b/cmd/ftl/cmd_init.go @@ -86,7 +86,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { scanner := bufio.NewScanner(f) for scanner.Scan() { - if strings.TrimSpace(scanner.Text()) == "**/_ftl" { + if strings.TrimSpace(scanner.Text()) == "**/.ftl" { return nil } } @@ -96,7 +96,7 @@ func updateGitIgnore(ctx context.Context, gitRoot string) error { } // append if not already present - if _, err = f.WriteString("**/_ftl\n"); err != nil { + if _, err = f.WriteString("**/.ftl\n"); err != nil { return err } diff --git a/cmd/ftl/cmd_new.go b/cmd/ftl/cmd_new.go index 831364a03c..1afa32ad1f 100644 --- a/cmd/ftl/cmd_new.go +++ b/cmd/ftl/cmd_new.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "github.com/TBD54566975/scaffolder" @@ -30,9 +31,10 @@ type newCmd struct { } type newGoCmd struct { - Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` - Dir string `arg:"" help:"Directory to initialize the module in."` - Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` + Replace map[string]string `short:"r" help:"Replace a module import path with a local path in the initialised FTL module." placeholder:"OLD=NEW,..." env:"FTL_INIT_GO_REPLACE"` + Dir string `arg:"" help:"Directory to initialize the module in."` + Name string `arg:"" help:"Name of the FTL module to create underneath the base directory."` + GoVersion string } type newKotlinCmd struct { @@ -58,6 +60,8 @@ func (i newGoCmd) Run(ctx context.Context) error { logger := log.FromContext(ctx) logger.Debugf("Creating FTL Go module %q in %s", name, path) + + i.GoVersion = runtime.Version()[2:] if err := scaffold(ctx, config.Hermit, goruntime.Files(), i.Dir, i, scaffolder.Exclude("^go.mod$")); err != nil { return err } diff --git a/common/moduleconfig/moduleconfig.go b/common/moduleconfig/moduleconfig.go index dddad51340..990e96b732 100644 --- a/common/moduleconfig/moduleconfig.go +++ b/common/moduleconfig/moduleconfig.go @@ -133,7 +133,7 @@ func setConfigDefaults(moduleDir string, config *ModuleConfig) error { case "go": if config.DeployDir == "" { - config.DeployDir = "_ftl" + config.DeployDir = ".ftl" } if len(config.Deploy) == 0 { config.Deploy = []string{"main"} diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index dbc72146ec..a539cf6ee4 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "ftl", - "version": "0.1.2", + "version": "0.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ftl", - "version": "0.1.2", + "version": "0.0.0", "dependencies": { "lookpath": "^1.2.2", "semver": "^7.6.0", diff --git a/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl b/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl similarity index 94% rename from go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl rename to go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl index 56b07af1f7..df0b6eca75 100644 --- a/go-runtime/compile/build-template/_ftl.tmpl/go/main/go.mod.tmpl +++ b/go-runtime/compile/build-template/.ftl.tmpl/go/main/go.mod.tmpl @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go b/go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl similarity index 100% rename from go-runtime/compile/build-template/_ftl.tmpl/go/main/main.go rename to go-runtime/compile/build-template/.ftl.tmpl/go/main/main.go.tmpl diff --git a/go-runtime/compile/build-template/go.work.tmpl b/go-runtime/compile/build-template/go.work.tmpl index 683cd1d340..cbd6143cf0 100644 --- a/go-runtime/compile/build-template/go.work.tmpl +++ b/go-runtime/compile/build-template/go.work.tmpl @@ -2,6 +2,8 @@ go {{ .GoVersion }} use ( . - _ftl/go/modules - _ftl/go/main +{{- range .SharedModulesPaths }} + {{ . }} +{{- end }} + .ftl/go/main ) diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 4a234a356d..d0276905e4 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -3,7 +3,6 @@ package compile import ( "context" "fmt" - "maps" "os" "path" "path/filepath" @@ -15,7 +14,7 @@ import ( "unicode" sets "github.com/deckarep/golang-set/v2" - gomaps "golang.org/x/exp/maps" + "golang.org/x/exp/maps" "golang.org/x/mod/modfile" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" @@ -35,12 +34,16 @@ import ( "github.com/TBD54566975/ftl/internal/reflect" ) +type MainWorkContext struct { + GoVersion string + SharedModulesPaths []string +} + type ExternalModuleContext struct { - ModuleDir string *schema.Schema GoVersion string FTLVersion string - Main string + Module *schema.Module Replacements []*modfile.Replace } @@ -53,13 +56,14 @@ type goVerb struct { } type mainModuleContext struct { - GoVersion string - FTLVersion string - Name string - Verbs []goVerb - Replacements []*modfile.Replace - SumTypes []goSumType - LocalSumTypes []goSumType + GoVersion string + FTLVersion string + Name string + SharedModulesPaths []string + Verbs []goVerb + Replacements []*modfile.Replace + SumTypes []goSumType + LocalSumTypes []goSumType } type goSumType struct { @@ -79,25 +83,14 @@ type ModifyFilesTransaction interface { End() error } -func (b ExternalModuleContext) NonMainModules() []*schema.Module { - modules := make([]*schema.Module, 0, len(b.Modules)) - for _, module := range b.Modules { - if module.Name == b.Main { - continue - } - modules = append(modules, module) - } - return modules -} - -const buildDirName = "_ftl" +const buildDirName = ".ftl" func buildDir(moduleDir string) string { return filepath.Join(moduleDir, buildDirName) } // Build the given module. -func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { +func Build(ctx context.Context, projectRootDir, moduleDir string, sch *schema.Schema, filesTransaction ModifyFilesTransaction) (err error) { if err := filesTransaction.Begin(); err != nil { return err } @@ -130,19 +123,27 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans funcs := maps.Clone(scaffoldFuncs) - logger.Debugf("Generating external modules") - if err := generateExternalModules(ExternalModuleContext{ - ModuleDir: moduleDir, - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Schema: sch, - Main: config.Module, - Replacements: replacements, - }); err != nil { - return fmt.Errorf("failed to generate external modules: %w", err) + buildDir := buildDir(moduleDir) + err = os.MkdirAll(buildDir, 0750) + if err != nil { + return fmt.Errorf("failed to create build directory: %w", err) + } + + var sharedModulesPaths []string + for _, mod := range sch.Modules { + if mod.Name == config.Module { + continue + } + sharedModulesPaths = append(sharedModulesPaths, filepath.Join(projectRootDir, buildDirName, "go", "modules", mod.Name)) + } + + if err := internal.ScaffoldZip(mainWorkTemplateFiles(), moduleDir, MainWorkContext{ + GoVersion: goModVersion, + SharedModulesPaths: sharedModulesPaths, + }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { + return fmt.Errorf("failed to scaffold zip: %w", err) } - buildDir := buildDir(moduleDir) logger.Debugf("Extracting schema") result, err := ExtractModuleSchema(config.Dir, sch) if err != nil { @@ -186,13 +187,14 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans goVerbs = append(goVerbs, goverb) } if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{ - GoVersion: goModVersion, - FTLVersion: ftlVersion, - Name: result.Module.Name, - Verbs: goVerbs, - Replacements: replacements, - SumTypes: getSumTypes(result.Module, sch, result.NativeNames), - LocalSumTypes: getLocalSumTypes(result.Module), + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Name: result.Module.Name, + SharedModulesPaths: sharedModulesPaths, + Verbs: goVerbs, + Replacements: replacements, + SumTypes: getSumTypes(result.Module, sch, result.NativeNames), + LocalSumTypes: getLocalSumTypes(result.Module), }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -200,7 +202,7 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans logger.Debugf("Tidying go.mod files") wg, wgctx := errgroup.WithContext(ctx) wg.Go(func() error { - if err := exec.Command(ctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { + if err := exec.Command(wgctx, log.Debug, moduleDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { return fmt.Errorf("%s: failed to tidy go.mod: %w", moduleDir, err) } return filesTransaction.ModifiedFiles(filepath.Join(moduleDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) @@ -212,13 +214,6 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans } return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) }) - modulesDir := filepath.Join(buildDir, "go", "modules") - wg.Go(func() error { - if err := exec.Command(wgctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(wgctx); err != nil { - return fmt.Errorf("%s: failed to tidy go.mod: %w", modulesDir, err) - } - return filesTransaction.ModifiedFiles(filepath.Join(modulesDir, "go.mod"), filepath.Join(moduleDir, "go.sum")) - }) if err := wg.Wait(); err != nil { return err } @@ -227,38 +222,92 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } -func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error { - goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod")) +// CleanStubs removes all generated stubs. +func CleanStubs(ctx context.Context, projectRoot string) error { + logger := log.FromContext(ctx) + logger.Debugf("Deleting all generated stubs") + sharedFtlDir := filepath.Join(projectRoot, buildDirName) + + // Wipe the modules directory to ensure we don't have any stale modules. + err := os.RemoveAll(sharedFtlDir) if err != nil { - return fmt.Errorf("failed to propagate replacements for library %q: %w", dir, err) + return fmt.Errorf("failed to remove %s: %w", sharedFtlDir, err) } + return nil +} + +// GenerateStubsForModules generates stubs for all modules in the schema. +func GenerateStubsForModules(ctx context.Context, projectRoot string, moduleConfigs []moduleconfig.ModuleConfig, sch *schema.Schema) error { + logger := log.FromContext(ctx) + logger.Debugf("Generating module stubs") + + sharedFtlDir := filepath.Join(projectRoot, buildDirName) + ftlVersion := "" if ftl.IsRelease(ftl.Version) { ftlVersion = ftl.Version } - return generateExternalModules(ExternalModuleContext{ - ModuleDir: dir, - GoVersion: goModFile.Go.Version, - FTLVersion: ftlVersion, - Schema: schema, - Replacements: replacements, - }) -} + for _, module := range sch.Modules { + var moduleConfig *moduleconfig.ModuleConfig + for _, mc := range moduleConfigs { + mcCopy := mc + if mc.Module == module.Name { + moduleConfig = &mcCopy + break + } + } -func generateExternalModules(context ExternalModuleContext) error { - // Wipe the modules directory to ensure we don't have any stale modules. - err := os.RemoveAll(filepath.Join(buildDir(context.ModuleDir), "go", "modules")) - if err != nil { - return fmt.Errorf("could not remove old external modules: %w", err) - } + var goModVersion string + var replacements []*modfile.Replace + var err error - funcs := maps.Clone(scaffoldFuncs) - err = internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) - if err != nil { - return fmt.Errorf("failed to scaffold external modules: %w", err) + // If there's no module config, use the go.mod file for the first config we find. + if moduleConfig == nil { + if len(moduleConfigs) > 0 { + _, goModVersion, err = updateGoModule(filepath.Join(moduleConfigs[0].Dir, "go.mod")) + if err != nil { + return err + } + } else { + // The best we can do here if we don't have a module to read from is to use the current Go version. + goModVersion = runtime.Version()[2:] + } + + replacements = []*modfile.Replace{} + } else { + replacements, goModVersion, err = updateGoModule(filepath.Join(moduleConfig.Dir, "go.mod")) + if err != nil { + return err + } + } + + goVersion := runtime.Version()[2:] + if semver.Compare("v"+goVersion, "v"+goModVersion) < 0 { + return fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion) + } + + context := ExternalModuleContext{ + Schema: sch, + GoVersion: goModVersion, + FTLVersion: ftlVersion, + Module: module, + Replacements: replacements, + } + + funcs := maps.Clone(scaffoldFuncs) + err = internal.ScaffoldZip(externalModuleTemplateFiles(), projectRoot, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) + if err != nil { + return fmt.Errorf("failed to scaffold zip: %w", err) + } + + modulesDir := filepath.Join(sharedFtlDir, "go", "modules", module.Name) + if err := exec.Command(ctx, log.Debug, modulesDir, "go", "mod", "tidy").RunBuffered(ctx); err != nil { + return fmt.Errorf("failed to tidy go.mod: %w", err) + } } + return nil } @@ -575,7 +624,7 @@ func getLocalSumTypes(module *schema.Module) []goSumType { } } } - out := gomaps.Values(sumTypes) + out := maps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) @@ -618,7 +667,7 @@ func getSumTypes(module *schema.Module, sch *schema.Schema, nativeNames NativeNa Variants: variants, } } - out := gomaps.Values(sumTypes) + out := maps.Values(sumTypes) slices.SortFunc(out, func(a, b goSumType) int { return strings.Compare(a.Discriminator, b.Discriminator) }) diff --git a/go-runtime/compile/devel.go b/go-runtime/compile/devel.go index c6f2f6270c..345309da66 100644 --- a/go-runtime/compile/devel.go +++ b/go-runtime/compile/devel.go @@ -8,9 +8,14 @@ import ( "github.com/TBD54566975/ftl/internal" ) +func mainWorkTemplateFiles() *zip.Reader { + return internal.ZipRelativeToCaller("main-work-template") +} + func externalModuleTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("external-module-template") } + func buildTemplateFiles() *zip.Reader { return internal.ZipRelativeToCaller("build-template") } diff --git a/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl similarity index 66% rename from go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl rename to go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl index a7c7c1b2de..657e8f20ab 100644 --- a/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl +++ b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/external_module.go.tmpl @@ -1,13 +1,13 @@ // Code generated by FTL. DO NOT EDIT. -package {{.Name}} +package {{.Module.Name}} import ( "context" -{{- range $import, $alias := (.|imports)}} +{{- range $import, $alias := (.Module | imports)}} {{if $alias}}{{$alias}} {{end}}"{{$import}}" {{- end}} -{{- $sumTypes := $ | sumTypes}} +{{- $sumTypes := (.Module | sumTypes)}} {{- if $sumTypes}} "github.com/TBD54566975/ftl/go-runtime/ftl/reflection" @@ -16,18 +16,18 @@ import ( var _ = context.Background -{{- range .Decls }} +{{- range .Module.Decls }} {{- if .IsExported}} {{if .Comments}} {{.Comments|comment -}} // {{- end}} {{- if is "Topic" .}} -var {{.Name|title}} = ftl.Topic[{{type $ .Event}}]("{{.Name}}") +var {{.Name|title}} = ftl.Topic[{{type $.Module .Event}}]("{{.Name}}") {{- else if and (is "Enum" .) .IsValueEnum}} {{- $enumName := .Name}} //ftl:enum -type {{.Name|title}} {{type $ .Type}} +type {{.Name|title}} {{type $.Module .Type}} const ( {{- range .Variants }} {{.Name|title}} {{$enumName}} = {{.Value|value}} @@ -38,14 +38,14 @@ const ( {{$enumInterfaceFuncName := enumInterfaceFunc . -}} type {{.Name|title}} interface { {{$enumInterfaceFuncName}}() } {{- range .Variants }} -{{if (or (basicType $ .) (isStandaloneEnumVariant .))}} -type {{.Name|title}} {{type $ .Value.Value}} +{{if (or (basicType $.Module .) (isStandaloneEnumVariant .))}} +type {{.Name|title}} {{type $.Module .Value.Value}} {{end}} func ({{.Name|title}}) {{$enumInterfaceFuncName}}() {} {{- end}} {{- else if is "TypeAlias" .}} //ftl:typealias -type {{.Name|title}} {{type $ .Type}} +type {{.Name|title}} {{type $.Module .Type}} {{- else if is "Data" .}} type {{.Name|title}} {{- if .TypeParameters}}[ @@ -54,25 +54,25 @@ type {{.Name|title}} {{- end -}} ]{{- end}} struct { {{- range .Fields}} - {{.Name|title}} {{type $ .Type}} `json:"{{.Name}}"` + {{.Name|title}} {{type $.Module .Type}} `json:"{{.Name}}"` {{- end}} } {{- else if is "Verb" .}} //ftl:verb -{{- if and (eq (type $ .Request) "ftl.Unit") (eq (type $ .Response) "ftl.Unit")}} +{{- if and (eq (type $.Module .Request) "ftl.Unit") (eq (type $.Module .Response) "ftl.Unit")}} func {{.Name|title}}(context.Context) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallEmpty()") } -{{- else if eq (type $ .Request) "ftl.Unit"}} -func {{.Name|title}}(context.Context) ({{type $ .Response}}, error) { +{{- else if eq (type $.Module .Request) "ftl.Unit"}} +func {{.Name|title}}(context.Context) ({{type $.Module .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSource()") } -{{- else if eq (type $ .Response) "ftl.Unit"}} -func {{.Name|title}}(context.Context, {{type $ .Request}}) error { +{{- else if eq (type $.Module .Response) "ftl.Unit"}} +func {{.Name|title}}(context.Context, {{type $.Module .Request}}) error { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.CallSink()") } {{- else}} -func {{.Name|title}}(context.Context, {{type $ .Request}}) ({{type $ .Response}}, error) { +func {{.Name|title}}(context.Context, {{type $.Module .Request}}) ({{type $.Module .Response}}, error) { panic("Verb stubs should not be called directly, instead use github.com/TBD54566975/ftl/runtime-go/ftl.Call()") } {{- end}} diff --git a/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl similarity index 80% rename from go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl rename to go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl index 2f1d05e47d..be3d760791 100644 --- a/go-runtime/compile/external-module-template/_ftl/go/modules/go.mod.tmpl +++ b/go-runtime/compile/external-module-template/.ftl/go/modules/{{ .Module.Name }}/go.mod.tmpl @@ -1,4 +1,4 @@ -module ftl +module ftl/{{ .Module.Name }} go {{ .GoVersion }} @@ -8,4 +8,4 @@ require github.com/TBD54566975/ftl v{{ .FTLVersion }} {{- range .Replacements }} replace {{ .Old }} => {{ .New }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/go-runtime/compile/external-module-template/go.mod b/go-runtime/compile/external-module-template/go.mod deleted file mode 100644 index 8da40da65a..0000000000 --- a/go-runtime/compile/external-module-template/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module exclude - -go 1.22.2 diff --git a/go-runtime/compile/external-module-template/go.work.tmpl b/go-runtime/compile/external-module-template/go.work.tmpl deleted file mode 100644 index 204b39b25e..0000000000 --- a/go-runtime/compile/external-module-template/go.work.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -go {{ .GoVersion }} - -use ( - . - _ftl/go/modules -) diff --git a/go-runtime/compile/main-work-template/go.work.tmpl b/go-runtime/compile/main-work-template/go.work.tmpl new file mode 100644 index 0000000000..1dbed607a3 --- /dev/null +++ b/go-runtime/compile/main-work-template/go.work.tmpl @@ -0,0 +1,8 @@ +go {{ .GoVersion }} + +use ( + . +{{- range .SharedModulesPaths }} + {{ . }} +{{- end }} +) diff --git a/go-runtime/compile/release.go b/go-runtime/compile/release.go index 8b909688cd..a258dd2170 100644 --- a/go-runtime/compile/release.go +++ b/go-runtime/compile/release.go @@ -8,12 +8,23 @@ import ( _ "embed" ) +//go:embed main-work-template.zip +var mainWorkTemplateBytes []byte + //go:embed external-module-template.zip var externalModuleTemplateBytes []byte //go:embed build-template.zip var buildTemplateBytes []byte +func mainWorkTemplateFiles() *zip.Reader { + zr, err := zip.NewReader(bytes.NewReader(mainWorkTemplateBytes), int64(len(mainWorkTemplateBytes))) + if err != nil { + panic(err) + } + return zr +} + func externalModuleTemplateFiles() *zip.Reader { zr, err := zip.NewReader(bytes.NewReader(externalModuleTemplateBytes), int64(len(externalModuleTemplateBytes))) if err != nil { diff --git a/go-runtime/compile/schema_test.go b/go-runtime/compile/schema_test.go index 8c974259bf..abbc0d83db 100644 --- a/go-runtime/compile/schema_test.go +++ b/go-runtime/compile/schema_test.go @@ -56,7 +56,7 @@ func TestExtractModuleSchema(t *testing.T) { assert.NoError(t, err) actual := schema.Normalise(r.Module) expected := `module one { - config configValue one.Config + config configValue one.Config secret secretValue String database postgres testDb @@ -184,6 +184,9 @@ func TestExtractModuleSchemaTwo(t *testing.T) { if testing.Short() { t.SkipNow() } + + assert.NoError(t, prebuildTestModule(t, "testdata/two")) + r, err := ExtractModuleSchema("testdata/two", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, r.Errors, nil) @@ -365,6 +368,9 @@ func TestExtractModulePubSub(t *testing.T) { if testing.Short() { t.SkipNow() } + + assert.NoError(t, prebuildTestModule(t, "testdata/pubsub")) + r, err := ExtractModuleSchema("testdata/pubsub", &schema.Schema{}) assert.NoError(t, err) assert.Equal(t, nil, r.Errors, "expected no schema errors") diff --git a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl index 372814bd3b..7d6b63f1d2 100644 --- a/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl +++ b/go-runtime/scaffolding/{{ .Name | camel | lower }}/go.mod.tmpl @@ -1,9 +1,9 @@ module ftl/{{ .Name }} -go 1.21 +go {{ .GoVersion }} require github.com/TBD54566975/ftl latest {{- range $old, $new := .Replace }} replace {{ $old }} => {{ $new }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/integration/harness.go b/integration/harness.go index d4c55fc955..6cb0af3369 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -17,6 +17,7 @@ import ( "connectrpc.com/connect" "github.com/alecthomas/assert/v2" "github.com/alecthomas/types/optional" + "github.com/otiai10/copy" ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1" "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect" @@ -73,11 +74,23 @@ func run(t *testing.T, ftlConfigPath string, startController bool, actions ...Ac assert.True(t, ok) if ftlConfigPath != "" { - // Use a path into the testdata directory instead of one relative to - // tmpDir. Otherwise we have a chicken and egg situation where the config - // can't be loaded until the module is copied over, and the config itself - // is used by FTL during startup. - t.Setenv("FTL_CONFIG", filepath.Join(cwd, "testdata", "go", ftlConfigPath)) + ftlConfigPath = filepath.Join(cwd, "testdata", "go", ftlConfigPath) + projectPath := filepath.Join(tmpDir, "ftl-project.toml") + + // Copy the specified FTL config to the temporary directory. + err = copy.Copy(ftlConfigPath, projectPath) + if err == nil { + t.Setenv("FTL_CONFIG", projectPath) + } else { + // Use a path into the testdata directory instead of one relative to + // tmpDir. Otherwise we have a chicken and egg situation where the config + // can't be loaded until the module is copied over, and the config itself + // is used by FTL during startup. + // Some tests still rely on this behavior, so we can't remove it entirely. + t.Logf("Failed to copy %s to %s: %s", ftlConfigPath, projectPath, err) + t.Setenv("FTL_CONFIG", ftlConfigPath) + } + } else { err = os.WriteFile(filepath.Join(tmpDir, "ftl-project.toml"), []byte(`name = "integration"`), 0644) assert.NoError(t, err)