diff --git a/buildengine/build.go b/buildengine/build.go index 6b83b8d3d1..9fb696c921 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -3,43 +3,64 @@ package buildengine import ( "context" "fmt" + "path/filepath" + "strings" "github.com/TBD54566975/ftl/backend/schema" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" + "github.com/TBD54566975/ftl/internal/slices" ) -// A Module is a ModuleConfig with its dependencies populated. -type Module struct { - moduleconfig.ModuleConfig - Dependencies []string -} - -// LoadModule loads a module from the given directory. -// -// A [Module] includes the module configuration as well as its dependencies -// extracted from source code. -func LoadModule(ctx context.Context, dir string) (Module, error) { - config, err := moduleconfig.LoadModuleConfig(dir) - if err != nil { - return Module{}, err +// Build a project in the given directory given the schema and project config. +// For a module, this will build the module. For an external library, this will build stubs for imported modules. +func Build(ctx context.Context, sch *schema.Schema, project Project) error { + switch project := project.(type) { + case Module: + return buildModule(ctx, sch, project) + case ExternalLibrary: + return buildExternalLibrary(ctx, sch, project) + default: + panic(fmt.Sprintf("unsupported project type: %T", project)) } - return UpdateDependencies(ctx, config) } -// Build a module in the given directory given the schema and module config. -func Build(ctx context.Context, sch *schema.Schema, module Module) error { +func buildModule(ctx context.Context, sch *schema.Schema, module Module) error { logger := log.FromContext(ctx).Scope(module.Module) ctx = log.ContextWithLogger(ctx, logger) + logger.Infof("Building module") switch module.Language { case "go": - return buildGo(ctx, sch, module) - + return buildGoModule(ctx, sch, module) case "kotlin": - return buildKotlin(ctx, sch, module) - + return buildKotlinModule(ctx, sch, module) default: return fmt.Errorf("unknown language %q", module.Language) } } + +func buildExternalLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error { + logger := log.FromContext(ctx).Scope(filepath.Base(lib.Dir)) + ctx = log.ContextWithLogger(ctx, logger) + + imported := slices.Map(sch.Modules, func(m *schema.Module) string { + return m.Name + }) + logger.Debugf("Generating stubs [%s] for %v", strings.Join(imported, ", "), lib) + + switch lib.Language { + case "go": + if err := buildGoLibrary(ctx, sch, lib); err != nil { + return err + } + case "kotlin": + if err := buildKotlinLibrary(ctx, sch, lib); err != nil { + return err + } + default: + return fmt.Errorf("unknown language %q for %s", lib.Language, lib) + } + + logger.Infof("Generated stubs [%s] for %v", strings.Join(imported, ", "), lib) + return nil +} diff --git a/buildengine/build_go.go b/buildengine/build_go.go index 31f1d31e75..107bd31880 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -8,9 +8,16 @@ import ( "github.com/TBD54566975/ftl/go-runtime/compile" ) -func buildGo(ctx context.Context, sch *schema.Schema, module Module) error { +func buildGoModule(ctx context.Context, sch *schema.Schema, module Module) error { if err := compile.Build(ctx, module.Dir, sch); err != nil { - return fmt.Errorf("failed to build module %s: %w", module.Module, err) + return fmt.Errorf("failed to build %q: %w", module, err) + } + return nil +} + +func buildGoLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error { + if err := compile.GenerateStubsForExternalLibrary(ctx, lib.Dir, sch); err != nil { + return fmt.Errorf("failed to generate stubs for %q: %w", lib, err) } return nil } diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index a04bcac1c5..e3f10776af 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -122,7 +122,7 @@ func Nothing(context.Context) error { } ` bctx := buildContext{ - moduleDir: "testdata/modules/another", + moduleDir: "testdata/projects/another", buildDir: "_ftl", sch: sch, } @@ -172,7 +172,7 @@ func Call(context.Context, Req) (Resp, error) { } ` bctx := buildContext{ - moduleDir: "testdata/modules/another", + moduleDir: "testdata/projects/another", buildDir: "_ftl", sch: sch, } @@ -182,11 +182,11 @@ func Call(context.Context, Req) (Resp, error) { } func TestExternalType(t *testing.T) { - moduleDir := "testdata/modules/external" + moduleDir := "testdata/projects/external" buildDir := "_ftl" ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{})) - module, err := LoadModule(ctx, moduleDir) + module, err := LoadModule(moduleDir) assert.NoError(t, err) sch := &schema.Schema{} diff --git a/buildengine/build_kotlin.go b/buildengine/build_kotlin.go index 80d9541809..86aa0d1711 100644 --- a/buildengine/build_kotlin.go +++ b/buildengine/build_kotlin.go @@ -23,14 +23,18 @@ import ( ) type externalModuleContext struct { - module Module + project Project *schema.Schema } func (e externalModuleContext) ExternalModules() []*schema.Module { + name := "" + if module, ok := e.project.(Module); ok { + name = module.Module + } modules := make([]*schema.Module, 0, len(e.Modules)) for _, module := range e.Modules { - if module.Name == e.module.Module { + if module.Name == name { continue } modules = append(modules, module) @@ -38,13 +42,13 @@ func (e externalModuleContext) ExternalModules() []*schema.Module { return modules } -func buildKotlin(ctx context.Context, sch *schema.Schema, module Module) error { +func buildKotlinModule(ctx context.Context, sch *schema.Schema, module Module) error { logger := log.FromContext(ctx) if err := SetPOMProperties(ctx, module.Dir); err != nil { return fmt.Errorf("unable to update ftl.version in %s: %w", module.Dir, err) } - if err := generateExternalModules(ctx, module, sch); err != nil { + if err := generateExternalModules(ctx, &module, sch); err != nil { return fmt.Errorf("unable to generate external modules for %s: %w", module.Module, err) } @@ -61,6 +65,13 @@ func buildKotlin(ctx context.Context, sch *schema.Schema, module Module) error { return nil } +func buildKotlinLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error { + if err := generateExternalModules(ctx, &lib, sch); err != nil { + return fmt.Errorf("unable to generate external modules for %v: %w", lib, err) + } + return nil +} + // SetPOMProperties updates the ftl.version properties in the // pom.xml file in the given base directory. func SetPOMProperties(ctx context.Context, baseDir string) error { @@ -121,18 +132,18 @@ exec java -cp "classes:$(cat classpath.txt)" xyz.block.ftl.main.MainKt return nil } -func generateExternalModules(ctx context.Context, module Module, sch *schema.Schema) error { +func generateExternalModules(ctx context.Context, project Project, sch *schema.Schema) error { logger := log.FromContext(ctx) - config := module.ModuleConfig funcs := maps.Clone(scaffoldFuncs) + config := project.Config() // Wipe the modules directory to ensure we don't have any stale modules. _ = os.RemoveAll(filepath.Join(config.Dir, "target", "generated-sources", "ftl")) logger.Debugf("Generating external modules") return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), config.Dir, externalModuleContext{ - module: module, - Schema: sch, + project: project, + Schema: sch, }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) } diff --git a/buildengine/build_kotlin_test.go b/buildengine/build_kotlin_test.go index a42d9328e8..2951c198e0 100644 --- a/buildengine/build_kotlin_test.go +++ b/buildengine/build_kotlin_test.go @@ -26,7 +26,7 @@ package ftl.test ` bctx := buildContext{ - moduleDir: "testdata/modules/echokotlin", + moduleDir: "testdata/projects/echokotlin", buildDir: "target", sch: sch, } @@ -149,7 +149,7 @@ data class TestResponse( ` bctx := buildContext{ - moduleDir: "testdata/modules/echokotlin", + moduleDir: "testdata/projects/echokotlin", buildDir: "target", sch: sch, } @@ -209,7 +209,7 @@ fun testVerb(context: Context, req: Request): ftl.builtin.Empty = throw NotImplementedError("Verb stubs should not be called directly, instead use context.call(::testVerb, ...)") ` bctx := buildContext{ - moduleDir: "testdata/modules/echokotlin", + moduleDir: "testdata/projects/echokotlin", buildDir: "target", sch: sch, } @@ -258,7 +258,7 @@ data class HttpResponse( class Empty ` bctx := buildContext{ - moduleDir: "testdata/modules/echokotlin", + moduleDir: "testdata/projects/echokotlin", buildDir: "target", sch: sch, } @@ -302,7 +302,7 @@ fun emptyVerb(context: Context, req: ftl.builtin.Empty): ftl.builtin.Empty = thr NotImplementedError("Verb stubs should not be called directly, instead use context.call(::emptyVerb, ...)") ` bctx := buildContext{ - moduleDir: "testdata/modules/echokotlin", + moduleDir: "testdata/projects/echokotlin", buildDir: "target", sch: sch, } @@ -380,7 +380,7 @@ fun nothing(context: Context): Unit = throw NotImplementedError("Verb stubs should not be called directly, instead use context.callEmpty(::nothing, ...)") ` bctx := buildContext{ - moduleDir: "testdata/modules/echokotlin", + moduleDir: "testdata/projects/echokotlin", buildDir: "target", sch: sch, } @@ -390,14 +390,13 @@ fun nothing(context: Context): Unit = throw } func TestKotlinExternalType(t *testing.T) { - moduleDir := "testdata/modules/externalkotlin" + moduleDir := "testdata/projects/externalkotlin" buildDir := "_ftl" ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{})) - module, err := LoadModule(ctx, moduleDir) + module, err := LoadModule(moduleDir) assert.NoError(t, err) - //create a logger that writes to a buffer.Bytes logBuffer := bytes.Buffer{} logger := log.Configure(&logBuffer, log.Config{}) ctx = log.ContextWithLogger(ctx, logger) diff --git a/buildengine/build_test.go b/buildengine/build_test.go index e959980e6c..0ccadcc9c9 100644 --- a/buildengine/build_test.go +++ b/buildengine/build_test.go @@ -2,11 +2,12 @@ package buildengine import ( "context" - "github.com/alecthomas/assert/v2" "os" "path/filepath" "testing" + "github.com/alecthomas/assert/v2" + "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/internal/log" ) @@ -26,7 +27,7 @@ func testBuild( ) { t.Helper() ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{})) - module, err := LoadModule(ctx, bctx.moduleDir) + module, err := LoadModule(bctx.moduleDir) assert.NoError(t, err) err = Build(ctx, bctx.sch, module) diff --git a/buildengine/deploy.go b/buildengine/deploy.go index 59fd6b4742..8e86c852f9 100644 --- a/buildengine/deploy.go +++ b/buildengine/deploy.go @@ -59,7 +59,7 @@ func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnl moduleSchema, err := loadProtoSchema(deployDir, module.ModuleConfig, replicas) if err != nil { - return fmt.Errorf("failed to load protobuf schema from %q: %w", module.ModuleConfig.Schema, err) + return fmt.Errorf("failed to load protobuf schema from %q: %w", module.Schema, err) } logger.Debugf("Uploading %d/%d files", len(gadResp.Msg.MissingDigests), len(files)) diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index f4b1c3fc35..de9c0e2e18 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -63,8 +63,8 @@ func TestDeploy(t *testing.T) { } ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{})) - modulePath := "testdata/modules/another" - module, err := LoadModule(ctx, modulePath) + modulePath := "testdata/projects/another" + module, err := LoadModule(modulePath) assert.NoError(t, err) // Build first to make sure the files are there. diff --git a/buildengine/deps.go b/buildengine/deps.go index c86964ae41..ff6b7c8e4c 100644 --- a/buildengine/deps.go +++ b/buildengine/deps.go @@ -14,34 +14,19 @@ import ( "strconv" "strings" - "golang.design/x/reflect" "golang.org/x/exp/maps" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" ) -// UpdateAllDependencies calls UpdateDependencies on each module in the list. -func UpdateAllDependencies(ctx context.Context, modules ...moduleconfig.ModuleConfig) ([]Module, error) { - out := []Module{} - for _, module := range modules { - updated, err := UpdateDependencies(ctx, module) - if err != nil { - return nil, err - } - out = append(out, updated) - } - return out, nil -} - -// UpdateDependencies finds the dependencies for an FTL module and returns a -// Module with those dependencies populated. -func UpdateDependencies(ctx context.Context, config moduleconfig.ModuleConfig) (Module, error) { +// UpdateDependencies finds the dependencies for a project and returns a +// Project with those dependencies populated. +func UpdateDependencies(ctx context.Context, project Project) (Project, error) { logger := log.FromContext(ctx) - logger.Debugf("Extracting dependencies for module %s", config.Module) - dependencies, err := extractDependencies(config) + logger.Debugf("Extracting dependencies for %s", project) + dependencies, err := extractDependencies(project) if err != nil { - return Module{}, err + return Project(&Module{}), err } containsBuiltin := false for _, dep := range dependencies { @@ -53,17 +38,23 @@ func UpdateDependencies(ctx context.Context, config moduleconfig.ModuleConfig) ( if !containsBuiltin { dependencies = append(dependencies, "builtin") } - out := reflect.DeepCopy(config) - return Module{ModuleConfig: out, Dependencies: dependencies}, nil + + out := project.CopyWithDependencies(dependencies) + return out, nil } -func extractDependencies(config moduleconfig.ModuleConfig) ([]string, error) { +func extractDependencies(project Project) ([]string, error) { + config := project.Config() + name := "" + if config, ok := project.(Module); ok { + name = config.Module + } switch config.Language { case "go": - return extractGoFTLImports(config.Module, config.Dir) + return extractGoFTLImports(name, config.Dir) case "kotlin": - return extractKotlinFTLImports(config.Module, config.Dir) + return extractKotlinFTLImports(name, config.Dir) default: return nil, fmt.Errorf("unsupported language: %s", config.Language) diff --git a/buildengine/deps_test.go b/buildengine/deps_test.go index 629f74918f..3e558521f7 100644 --- a/buildengine/deps_test.go +++ b/buildengine/deps_test.go @@ -6,14 +6,26 @@ import ( "github.com/alecthomas/assert/v2" ) -func TestExtractDepsGo(t *testing.T) { - deps, err := extractGoFTLImports("test", "testdata/modules/alpha") +func TestExtractModuleDepsGo(t *testing.T) { + deps, err := extractGoFTLImports("test", "testdata/projects/alpha") assert.NoError(t, err) assert.Equal(t, []string{"another", "other"}, deps) } -func TestExtractDepsKotlin(t *testing.T) { - deps, err := extractKotlinFTLImports("test", "testdata/modules/alphakotlin") +func TestExtractModuleDepsKotlin(t *testing.T) { + deps, err := extractKotlinFTLImports("test", "testdata/projects/alphakotlin") assert.NoError(t, err) assert.Equal(t, []string{"builtin", "other"}, deps) } + +func TestExtractLibraryDepsGo(t *testing.T) { + deps, err := extractGoFTLImports("test", "testdata/projects/lib") + assert.NoError(t, err) + assert.Equal(t, []string{"alpha"}, deps) +} + +func TestExtractLibraryDepsKotlin(t *testing.T) { + deps, err := extractKotlinFTLImports("test", "testdata/projects/libkotlin") + assert.NoError(t, err) + assert.Equal(t, []string{"builtin", "echo"}, deps) +} diff --git a/buildengine/discover.go b/buildengine/discover.go index 49985a8c44..c411abd1cd 100644 --- a/buildengine/discover.go +++ b/buildengine/discover.go @@ -7,13 +7,45 @@ import ( "path/filepath" "sort" - "github.com/TBD54566975/ftl/common/moduleconfig" + "github.com/TBD54566975/ftl/internal/log" ) -// DiscoverModules recursively loads all modules under the given directories. +// DiscoverProjects recursively loads all modules under the given directories +// (or if none provided, the current working directory is used) and external +// libraries in externalLibDirs. +func DiscoverProjects(ctx context.Context, moduleDirs []string, externalLibDirs []string, stopOnError bool) ([]Project, error) { + out := []Project{} + logger := log.FromContext(ctx) + + modules, err := discoverModules(moduleDirs...) + if err != nil { + logger.Tracef("error discovering modules: %v", err) + if stopOnError { + return nil, err + } + } else { + for _, module := range modules { + out = append(out, Project(module)) + } + } + for _, dir := range externalLibDirs { + lib, err := LoadExternalLibrary(dir) + if err != nil { + logger.Tracef("error discovering external library: %v", err) + if stopOnError { + return nil, err + } + } else { + out = append(out, Project(lib)) + } + } + return out, nil +} + +// discoverModules recursively loads all modules under the given directories. // // If no directories are provided, the current working directory is used. -func DiscoverModules(ctx context.Context, dirs ...string) ([]moduleconfig.ModuleConfig, error) { +func discoverModules(dirs ...string) ([]Module, error) { if len(dirs) == 0 { cwd, err := os.Getwd() if err != nil { @@ -21,18 +53,18 @@ func DiscoverModules(ctx context.Context, dirs ...string) ([]moduleconfig.Module } dirs = []string{cwd} } - out := []moduleconfig.ModuleConfig{} + out := []Module{} for _, dir := range dirs { err := WalkDir(dir, func(path string, d fs.DirEntry) error { if filepath.Base(path) != "ftl.toml" { return nil } moduleDir := filepath.Dir(path) - config, err := moduleconfig.LoadModuleConfig(moduleDir) + module, err := LoadModule(moduleDir) if err != nil { return err } - out = append(out, config) + out = append(out, module) return ErrSkip }) if err != nil { @@ -40,7 +72,7 @@ func DiscoverModules(ctx context.Context, dirs ...string) ([]moduleconfig.Module } } sort.Slice(out, func(i, j int) bool { - return out[i].Module < out[j].Module + return out[i].Config().Key < out[j].Config().Key }) return out, nil } diff --git a/buildengine/discover_test.go b/buildengine/discover_test.go index 0e558772cf..67e64bb93a 100644 --- a/buildengine/discover_test.go +++ b/buildengine/discover_test.go @@ -12,96 +12,116 @@ import ( func TestDiscoverModules(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - modules, err := DiscoverModules(ctx, "testdata/modules") + projects, err := DiscoverProjects(ctx, []string{"testdata/projects"}, []string{"testdata/projects/lib", "testdata/projects/libkotlin"}, true) assert.NoError(t, err) - expected := []moduleconfig.ModuleConfig{ - { - Dir: "testdata/modules/alpha", - Language: "go", - Realm: "home", - Module: "alpha", - Deploy: []string{"main"}, - DeployDir: "_ftl", - Schema: "schema.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum"}, - }, - { - Dir: "testdata/modules/another", - Language: "go", - Realm: "home", - Module: "another", - Deploy: []string{"main"}, - DeployDir: "_ftl", - Schema: "schema.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum"}, - }, - { - Dir: "testdata/modules/echokotlin", - Language: "kotlin", - Realm: "home", - Module: "echo", - Build: "mvn -B compile", - Deploy: []string{ - "main", - "classes", - "dependency", - "classpath.txt", + expected := []Project{ + Module{ + ModuleConfig: moduleconfig.ModuleConfig{ + Dir: "testdata/projects/alpha", + Language: "go", + Realm: "home", + Module: "alpha", + Deploy: []string{"main"}, + DeployDir: "_ftl", + Schema: "schema.pb", + Watch: []string{"**/*.go", "go.mod", "go.sum"}, }, - DeployDir: "target", - Schema: "schema.pb", - Watch: []string{ - "pom.xml", - "src/**", - "target/generated-sources", + }, + Module{ + ModuleConfig: moduleconfig.ModuleConfig{ + Dir: "testdata/projects/another", + Language: "go", + Realm: "home", + Module: "another", + Deploy: []string{"main"}, + DeployDir: "_ftl", + Schema: "schema.pb", + Watch: []string{"**/*.go", "go.mod", "go.sum"}, }, }, - { - Dir: "testdata/modules/external", - Language: "go", - Realm: "home", - Module: "external", - Build: "", - Deploy: []string{ - "main", + Module{ + ModuleConfig: moduleconfig.ModuleConfig{ + Dir: "testdata/projects/echokotlin", + Language: "kotlin", + Realm: "home", + Module: "echo", + Build: "mvn -B compile", + Deploy: []string{ + "main", + "classes", + "dependency", + "classpath.txt", + }, + DeployDir: "target", + Schema: "schema.pb", + Watch: []string{ + "pom.xml", + "src/**", + "target/generated-sources", + }, }, - DeployDir: "_ftl", - Schema: "schema.pb", - Watch: []string{ - "**/*.go", - "go.mod", - "go.sum", + }, + Module{ + ModuleConfig: moduleconfig.ModuleConfig{ + Dir: "testdata/projects/external", + Language: "go", + Realm: "home", + Module: "external", + Build: "", + Deploy: []string{ + "main", + }, + DeployDir: "_ftl", + Schema: "schema.pb", + Watch: []string{ + "**/*.go", + "go.mod", + "go.sum", + }, }, }, - { - Dir: "testdata/modules/externalkotlin", - Language: "kotlin", - Realm: "home", - Module: "externalkotlin", - Build: "mvn -B compile", - Deploy: []string{ - "main", - "classes", - "dependency", - "classpath.txt", + Module{ + ModuleConfig: moduleconfig.ModuleConfig{ + Dir: "testdata/projects/externalkotlin", + Language: "kotlin", + Realm: "home", + Module: "externalkotlin", + Build: "mvn -B compile", + Deploy: []string{ + "main", + "classes", + "dependency", + "classpath.txt", + }, + DeployDir: "target", + Schema: "schema.pb", + Watch: []string{ + "pom.xml", + "src/**", + "target/generated-sources", + }, }, - DeployDir: "target", - Schema: "schema.pb", - Watch: []string{ - "pom.xml", - "src/**", - "target/generated-sources", + }, + Module{ + ModuleConfig: moduleconfig.ModuleConfig{ + Dir: "testdata/projects/other", + Language: "go", + Realm: "home", + Module: "other", + Deploy: []string{"main"}, + DeployDir: "_ftl", + Schema: "schema.pb", + Watch: []string{"**/*.go", "go.mod", "go.sum"}, }, }, - { - Dir: "testdata/modules/other", - Language: "go", - Realm: "home", - Module: "other", - Deploy: []string{"main"}, - DeployDir: "_ftl", - Schema: "schema.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum"}, + ExternalLibrary{ + Dir: "testdata/projects/lib", + Language: "go", + }, + ExternalLibrary{ + Dir: "testdata/projects/libkotlin", + Language: "kotlin", }, } - assert.Equal(t, expected, modules) + assert.Equal(t, expected, projects) } diff --git a/buildengine/engine.go b/buildengine/engine.go index f1f843a4ea..708aac1899 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -32,8 +32,9 @@ type schemaChange struct { // Engine for building a set of modules. type Engine struct { client ftlv1connect.ControllerServiceClient - modules map[string]Module - dirs []string + projects map[ProjectKey]Project + moduleDirs []string + externalDirs []string controllerSchema *xsync.MapOf[string, *schema.Module] schemaChanges *pubsub.Topic[schemaChange] cancel func() @@ -55,12 +56,13 @@ func Parallelism(n int) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, dirs []string, options ...Option) (*Engine, error) { +func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, externalDirs []string, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, - dirs: dirs, - modules: map[string]Module{}, + moduleDirs: moduleDirs, + externalDirs: externalDirs, + projects: map[ProjectKey]Project{}, controllerSchema: xsync.NewMapOf[string, *schema.Module](), schemaChanges: pubsub.New[schemaChange](), parallelism: runtime.NumCPU(), @@ -71,17 +73,19 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, dirs e.controllerSchema.Store("builtin", schema.Builtins()) ctx, cancel := context.WithCancel(ctx) e.cancel = cancel - configs, err := DiscoverModules(ctx, dirs...) - if err != nil { - return nil, err - } - modules, err := UpdateAllDependencies(ctx, configs...) + + projects, err := DiscoverProjects(ctx, moduleDirs, externalDirs, true) if err != nil { - return nil, err + return nil, fmt.Errorf("could not find projects: %w", err) } - for _, module := range modules { - e.modules[module.Module] = module + for _, project := range projects { + project, err = UpdateDependencies(ctx, project) + if err != nil { + return nil, err + } + e.projects[project.Config().Key] = project } + if client == nil { return e, nil } @@ -138,29 +142,29 @@ func (e *Engine) Close() error { // // If no modules are provided, the entire graph is returned. An error is returned if // any dependencies are missing. -func (e *Engine) Graph(modules ...string) (map[string][]string, error) { +func (e *Engine) Graph(projects ...ProjectKey) (map[string][]string, error) { out := map[string][]string{} - if len(modules) == 0 { - modules = maps.Keys(e.modules) + if len(projects) == 0 { + projects = maps.Keys(e.projects) } - for _, name := range modules { - if err := e.buildGraph(name, out); err != nil { + for _, key := range projects { + if err := e.buildGraph(string(key), out); err != nil { return nil, err } } return out, nil } -func (e *Engine) buildGraph(name string, out map[string][]string) error { +func (e *Engine) buildGraph(key string, out map[string][]string) error { var deps []string - if module, ok := e.modules[name]; ok { - deps = module.Dependencies - } else if sch, ok := e.controllerSchema.Load(name); ok { + if project, ok := e.projects[ProjectKey(key)]; ok { + deps = project.Config().Dependencies + } else if sch, ok := e.controllerSchema.Load(key); ok { deps = sch.Imports() } else { - return fmt.Errorf("module %q not found", name) + return fmt.Errorf("module %q not found", key) } - out[name] = deps + out[key] = deps for _, dep := range deps { if err := e.buildGraph(dep, out); err != nil { return err @@ -220,7 +224,7 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration defer e.schemaChanges.Unsubscribe(schemaChanges) watchEvents := make(chan WatchEvent, 128) - watch := Watch(ctx, period, e.dirs...) + watch := Watch(ctx, period, e.moduleDirs, e.externalDirs) watch.Subscribe(watchEvents) defer watch.Unsubscribe(watchEvents) defer watch.Close() @@ -232,25 +236,29 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration return ctx.Err() case event := <-watchEvents: switch event := event.(type) { - case WatchEventModuleAdded: - if _, exists := e.modules[event.Module.Module]; !exists { - e.modules[event.Module.Module] = event.Module - err := e.buildAndDeploy(ctx, 1, true, event.Module.Module) + case WatchEventProjectAdded: + config := event.Project.Config() + if _, exists := e.projects[config.Key]; !exists { + e.projects[config.Key] = event.Project + err := e.buildAndDeploy(ctx, 1, true, config.Key) if err != nil { - logger.Errorf(err, "deploy %s failed", event.Module.Module) + logger.Errorf(err, "deploy %s failed", config.Key) } } - case WatchEventModuleRemoved: - err := teminateModuleDeployment(ctx, e.client, event.Module.Module) - if err != nil { - logger.Errorf(err, "terminate %s failed", event.Module.Module) + case WatchEventProjectRemoved: + config := event.Project.Config() + if module, ok := event.Project.(Module); ok { + err := teminateModuleDeployment(ctx, e.client, module.Module) + if err != nil { + logger.Errorf(err, "terminate %s failed", module.Module) + } } - delete(e.modules, event.Module.Module) - - case WatchEventModuleChanged: - err := e.buildAndDeploy(ctx, 1, true, event.Module.Module) + delete(e.projects, config.Key) + case WatchEventProjectChanged: + config := event.Project.Config() + err := e.buildAndDeploy(ctx, 1, true, config.Key) if err != nil { - logger.Errorf(err, "deploy %s failed", event.Module.Module) + logger.Errorf(err, "build and deploy failed for %v: %v", event.Project, err) } } case change := <-schemaChanges: @@ -270,10 +278,12 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration } moduleHashes[change.Name] = hash - modulesToDeploy := e.getDependentModules(change.Name) - if len(modulesToDeploy) > 0 { - logger.Infof("%s's schema changed; redeploying %s", change.Name, strings.Join(modulesToDeploy, ", ")) - err = e.buildAndDeploy(ctx, 1, true, modulesToDeploy...) + + dependentProjectKeys := e.getDependentProjectKeys(change.Name) + if len(dependentProjectKeys) > 0 { + //TODO: inaccurate log message for ext libs + logger.Infof("%s's schema changed; processing %s", change.Name, strings.Join(StringsFromProjectKeys(dependentProjectKeys), ", ")) + err = e.buildAndDeploy(ctx, 1, true, dependentProjectKeys...) if err != nil { logger.Errorf(err, "deploy %s failed", change.Name) } @@ -293,52 +303,56 @@ func computeModuleHash(module *schema.Module) ([]byte, error) { return hasher.Sum(nil), nil } -func (e *Engine) getDependentModules(moduleName string) []string { - dependentModules := map[string]bool{} - for key, module := range e.modules { - for _, dep := range module.Dependencies { - if dep == moduleName { - dependentModules[key] = true +func (e *Engine) getDependentProjectKeys(name string) []ProjectKey { + dependentProjectKeys := map[ProjectKey]bool{} + for k, project := range e.projects { + for _, dep := range project.Config().Dependencies { + if dep == name { + dependentProjectKeys[k] = true } } } - - return maps.Keys(dependentModules) + return maps.Keys(dependentProjectKeys) } -func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDeployOnline bool, modules ...string) error { - if len(modules) == 0 { - modules = maps.Keys(e.modules) +func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDeployOnline bool, projects ...ProjectKey) error { + if len(projects) == 0 { + projects = maps.Keys(e.projects) } - deployQueue := make(chan Module, len(modules)) + deployQueue := make(chan Project, len(projects)) wg, ctx := errgroup.WithContext(ctx) - // Build all modules and enqueue them for deployment. + // Build all projects and enqueue the modules for deployment. wg.Go(func() error { defer close(deployQueue) - return e.buildWithCallback(ctx, func(ctx context.Context, module Module) error { - select { - case deployQueue <- module: - return nil - case <-ctx.Done(): - return ctx.Err() + return e.buildWithCallback(ctx, func(ctx context.Context, project Project) error { + if _, ok := project.(Module); ok { + select { + case deployQueue <- project: + return nil + case <-ctx.Done(): + return ctx.Err() + } } - }, modules...) + return nil + }, projects...) }) // Process deployment queue. - for i := 0; i < len(modules); i++ { + for i := 0; i < len(projects); i++ { wg.Go(func() error { for { select { - case module, ok := <-deployQueue: + case project, ok := <-deployQueue: if !ok { return nil } - if err := Deploy(ctx, module, replicas, waitForDeployOnline, e.client); err != nil { - return err + if module, ok := project.(Module); ok { + if err := Deploy(ctx, module, replicas, waitForDeployOnline, e.client); err != nil { + return err + } } case <-ctx.Done(): return ctx.Err() @@ -346,36 +360,35 @@ func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDepl } }) } - return wg.Wait() } -type buildCallback func(ctx context.Context, module Module) error +type buildCallback func(ctx context.Context, project Project) error -func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, modules ...string) error { - mustBuild := map[string]bool{} - if len(modules) == 0 { - modules = maps.Keys(e.modules) +func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, projects ...ProjectKey) error { + mustBuild := map[ProjectKey]bool{} + if len(projects) == 0 { + projects = maps.Keys(e.projects) } - for _, name := range modules { - module, ok := e.modules[name] + for _, key := range projects { + project, ok := e.projects[key] if !ok { - return fmt.Errorf("module %q not found", module) + return fmt.Errorf("project %q not found", key) } // Update dependencies before building. var err error - module, err = UpdateDependencies(ctx, module.ModuleConfig) + project, err = UpdateDependencies(ctx, project) if err != nil { return err } - e.modules[name] = module - mustBuild[name] = true + e.projects[key] = project + mustBuild[key] = true } - graph, err := e.Graph(modules...) + graph, err := e.Graph(projects...) if err != nil { return err } - built := map[string]*schema.Module{ + builtModules := map[string]*schema.Module{ "builtin": schema.Builtins(), } @@ -385,70 +398,95 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, schemas := make(chan *schema.Module, len(group)) wg, ctx := errgroup.WithContext(ctx) wg.SetLimit(e.parallelism) - for _, name := range group { + for _, keyStr := range group { + key := ProjectKey(keyStr) wg.Go(func() error { - if mustBuild[name] { - err := e.build(ctx, name, built, schemas) - if err == nil && callback != nil { - return callback(ctx, e.modules[name]) - } - return err + if !mustBuild[key] { + return e.mustSchema(ctx, key, builtModules, schemas) + } + + err := e.build(ctx, key, builtModules, schemas) + if err == nil && callback != nil { + return callback(ctx, e.projects[key]) } - return e.mustSchema(ctx, name, built, schemas) + return err }) } err := wg.Wait() if err != nil { return err } - // Now this group is built, collect al the schemas. + // Now this group is built, collect all the schemas. close(schemas) for sch := range schemas { - built[sch.Name] = sch + builtModules[sch.Name] = sch } } + return nil } // Publish either the schema from the FTL controller, or from a local build. -func (e *Engine) mustSchema(ctx context.Context, name string, built map[string]*schema.Module, schemas chan<- *schema.Module) error { - if sch, ok := e.controllerSchema.Load(name); ok { +func (e *Engine) mustSchema(ctx context.Context, key ProjectKey, builtModules map[string]*schema.Module, schemas chan<- *schema.Module) error { + if sch, ok := e.controllerSchema.Load(string(key)); ok { schemas <- sch return nil } - return e.build(ctx, name, built, schemas) + return e.build(ctx, key, builtModules, schemas) } // Build a module and publish its schema. // // Assumes that all dependencies have been built and are available in "built". -func (e *Engine) build(ctx context.Context, name string, built map[string]*schema.Module, schemas chan<- *schema.Module) error { +func (e *Engine) build(ctx context.Context, key ProjectKey, builtModules map[string]*schema.Module, schemas chan<- *schema.Module) error { + project, ok := e.projects[key] + if !ok { + return fmt.Errorf("project %q not found", key) + } + combined := map[string]*schema.Module{} - gatherShemas(built, e.modules, e.modules[name], combined) + if err := e.gatherSchemas(builtModules, project, combined); err != nil { + return err + } sch := &schema.Schema{Modules: maps.Values(combined)} - module := e.modules[name] - err := Build(ctx, sch, module) + + err := Build(ctx, sch, project) if err != nil { return err } - moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(module.Dir, module.DeployDir, module.Schema)) - if err != nil { - return fmt.Errorf("load schema %s: %w", name, err) + if module, ok := project.(Module); ok { + moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(module.Dir, module.DeployDir, module.Schema)) + if err != nil { + return fmt.Errorf("could not load schema for %s: %w", module, err) + } + schemas <- moduleSchema } - schemas <- moduleSchema return nil } -// Construct a combined schema for a module and its transitive dependencies. -func gatherShemas( +// Construct a combined schema for a project and its transitive dependencies. +func (e *Engine) gatherSchemas( moduleSchemas map[string]*schema.Module, - modules map[string]Module, - module Module, + project Project, out map[string]*schema.Module, -) { - for _, dep := range modules[module.Module].Dependencies { +) error { + latestModule, ok := e.projects[project.Config().Key] + if !ok { + latestModule = project + } + for _, dep := range latestModule.Config().Dependencies { out[dep] = moduleSchemas[dep] - gatherShemas(moduleSchemas, modules, modules[dep], out) + if dep != "builtin" { + depModule, ok := e.projects[ProjectKey(dep)] + // TODO: should we be gathering schemas from dependencies without a project? + // This can happen if the schema is loaded from the controller + if ok { + if err := e.gatherSchemas(moduleSchemas, depModule, out); err != nil { + return err + } + } + } } + return nil } diff --git a/buildengine/engine_test.go b/buildengine/engine_test.go index 7f977ce5ff..e4252d894f 100644 --- a/buildengine/engine_test.go +++ b/buildengine/engine_test.go @@ -13,7 +13,7 @@ import ( func TestEngine(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/modules/alpha", "testdata/modules/another"}) + engine, err := buildengine.New(ctx, nil, []string{"testdata/projects/alpha", "testdata/projects/another"}, nil) assert.NoError(t, err) defer engine.Close() diff --git a/buildengine/filehash.go b/buildengine/filehash.go index 8c3374ff98..53a3e85d01 100644 --- a/buildengine/filehash.go +++ b/buildengine/filehash.go @@ -9,8 +9,6 @@ import ( "path/filepath" "github.com/bmatcuk/doublestar/v4" - - "github.com/TBD54566975/ftl/common/moduleconfig" ) type FileChangeType rune @@ -65,7 +63,9 @@ func CompareFileHashes(oldFiles, newFiles FileHashes) (FileChangeType, string, b // ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in // the given directory. -func ComputeFileHashes(config moduleconfig.ModuleConfig) (FileHashes, error) { +func ComputeFileHashes(module Project) (FileHashes, error) { + config := module.Config() + fileHashes := make(FileHashes) err := WalkDir(config.Dir, func(srcPath string, entry fs.DirEntry) error { for _, pattern := range config.Watch { diff --git a/buildengine/project.go b/buildengine/project.go new file mode 100644 index 0000000000..cb322886ac --- /dev/null +++ b/buildengine/project.go @@ -0,0 +1,157 @@ +package buildengine + +import ( + "fmt" + "os" + "path/filepath" + + "golang.design/x/reflect" + + "github.com/TBD54566975/ftl/common/moduleconfig" +) + +// Project models FTL modules and external libraries and is used to manage dependencies within the build engine +// +//sumtype:decl +type Project interface { + sealed() + + Config() ProjectConfig + CopyWithDependencies([]string) Project + String() string +} + +type ProjectConfig struct { + Key ProjectKey + Dir string + Language string + Watch []string + Dependencies []string +} + +var _ = (Project)(Module{}) +var _ = (Project)(ExternalLibrary{}) + +// Module represents an FTL module in the build engine +type Module struct { + moduleconfig.ModuleConfig + Dependencies []string +} + +func (m Module) sealed() {} + +func (m Module) Config() ProjectConfig { + return ProjectConfig{ + Key: ProjectKey(m.ModuleConfig.Module), + Dir: m.ModuleConfig.Dir, + Language: m.ModuleConfig.Language, + Watch: m.ModuleConfig.Watch, + Dependencies: m.Dependencies, + } +} + +func (m Module) CopyWithDependencies(dependencies []string) Project { + module := reflect.DeepCopy(m) + module.Dependencies = dependencies + return Project(module) +} + +func (m Module) String() string { + return "module " + m.ModuleConfig.Module +} + +// ExternalLibrary represents a library that makes use of FTL modules, but is not itself an FTL module +type ExternalLibrary struct { + Dir string + Language string + Dependencies []string +} + +func (e ExternalLibrary) sealed() {} + +func (e ExternalLibrary) Config() ProjectConfig { + var watch []string + switch e.Language { + case "go": + watch = []string{"**/*.go", "go.mod", "go.sum"} + case "kotlin": + watch = []string{"pom.xml", "src/**", "target/generated-sources"} + default: + panic(fmt.Sprintf("unknown language %T", e.Language)) + } + + return ProjectConfig{ + Key: ProjectKey("lib:" + e.Dir), + Dir: e.Dir, + Language: e.Language, + Watch: watch, + Dependencies: e.Dependencies, + } +} + +func (e ExternalLibrary) CopyWithDependencies(dependencies []string) Project { + lib := reflect.DeepCopy(e) + lib.Dependencies = dependencies + return Project(lib) +} + +func (e ExternalLibrary) String() string { + return "library " + e.Dir +} + +// ProjectKey is a unique identifier for the project (ie: a module name or a library path) +// It is used to: +// - build the dependency graph +// - map changes in the file system to the project +type ProjectKey string + +func StringsFromProjectKeys(keys []ProjectKey) []string { + strs := make([]string, len(keys)) + for i, key := range keys { + strs[i] = string(key) + } + return strs +} + +func ProjectKeysFromModuleNames(names []string) []ProjectKey { + keys := make([]ProjectKey, len(names)) + for i, str := range names { + keys[i] = ProjectKey(str) + } + return keys +} + +// LoadModule loads a module from the given directory. +func LoadModule(dir string) (Module, error) { + config, err := moduleconfig.LoadModuleConfig(dir) + if err != nil { + return Module{}, err + } + module := Module{ModuleConfig: config} + return module, nil +} + +func LoadExternalLibrary(dir string) (ExternalLibrary, error) { + lib := ExternalLibrary{ + Dir: dir, + } + + goModPath := filepath.Join(dir, "go.mod") + pomPath := filepath.Join(dir, "pom.xml") + if _, err := os.Stat(goModPath); err == nil { + lib.Language = "go" + } else if !os.IsNotExist(err) { + return ExternalLibrary{}, err + } else { + if _, err = os.Stat(pomPath); err == nil { + lib.Language = "kotlin" + } else if !os.IsNotExist(err) { + return ExternalLibrary{}, err + } + } + if lib.Language == "" { + return ExternalLibrary{}, fmt.Errorf("could not autodetect language: no go.mod or pom.xml found in %s", dir) + } + + return lib, nil +} diff --git a/buildengine/testdata/modules/alpha/alpha.go b/buildengine/testdata/projects/alpha/alpha.go similarity index 100% rename from buildengine/testdata/modules/alpha/alpha.go rename to buildengine/testdata/projects/alpha/alpha.go diff --git a/buildengine/testdata/modules/alpha/ftl.toml b/buildengine/testdata/projects/alpha/ftl.toml similarity index 100% rename from buildengine/testdata/modules/alpha/ftl.toml rename to buildengine/testdata/projects/alpha/ftl.toml diff --git a/buildengine/testdata/modules/alpha/go.mod b/buildengine/testdata/projects/alpha/go.mod similarity index 100% rename from buildengine/testdata/modules/alpha/go.mod rename to buildengine/testdata/projects/alpha/go.mod diff --git a/buildengine/testdata/modules/alpha/go.sum b/buildengine/testdata/projects/alpha/go.sum similarity index 100% rename from buildengine/testdata/modules/alpha/go.sum rename to buildengine/testdata/projects/alpha/go.sum diff --git a/buildengine/testdata/modules/alpha/pkg/pkg.go b/buildengine/testdata/projects/alpha/pkg/pkg.go similarity index 100% rename from buildengine/testdata/modules/alpha/pkg/pkg.go rename to buildengine/testdata/projects/alpha/pkg/pkg.go diff --git a/buildengine/testdata/modules/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt b/buildengine/testdata/projects/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt similarity index 100% rename from buildengine/testdata/modules/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt rename to buildengine/testdata/projects/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt diff --git a/buildengine/testdata/modules/another/another.go b/buildengine/testdata/projects/another/another.go similarity index 100% rename from buildengine/testdata/modules/another/another.go rename to buildengine/testdata/projects/another/another.go diff --git a/buildengine/testdata/modules/another/ftl.toml b/buildengine/testdata/projects/another/ftl.toml similarity index 100% rename from buildengine/testdata/modules/another/ftl.toml rename to buildengine/testdata/projects/another/ftl.toml diff --git a/buildengine/testdata/modules/another/go.mod b/buildengine/testdata/projects/another/go.mod similarity index 100% rename from buildengine/testdata/modules/another/go.mod rename to buildengine/testdata/projects/another/go.mod diff --git a/buildengine/testdata/modules/another/go.sum b/buildengine/testdata/projects/another/go.sum similarity index 100% rename from buildengine/testdata/modules/another/go.sum rename to buildengine/testdata/projects/another/go.sum diff --git a/buildengine/testdata/modules/echokotlin/ftl.toml b/buildengine/testdata/projects/echokotlin/ftl.toml similarity index 100% rename from buildengine/testdata/modules/echokotlin/ftl.toml rename to buildengine/testdata/projects/echokotlin/ftl.toml diff --git a/buildengine/testdata/modules/echokotlin/pom.xml b/buildengine/testdata/projects/echokotlin/pom.xml similarity index 100% rename from buildengine/testdata/modules/echokotlin/pom.xml rename to buildengine/testdata/projects/echokotlin/pom.xml diff --git a/buildengine/testdata/modules/echokotlin/src/main/kotlin/ftl/echo/Echo.kt b/buildengine/testdata/projects/echokotlin/src/main/kotlin/ftl/echo/Echo.kt similarity index 100% rename from buildengine/testdata/modules/echokotlin/src/main/kotlin/ftl/echo/Echo.kt rename to buildengine/testdata/projects/echokotlin/src/main/kotlin/ftl/echo/Echo.kt diff --git a/buildengine/testdata/modules/external/external.go b/buildengine/testdata/projects/external/external.go similarity index 100% rename from buildengine/testdata/modules/external/external.go rename to buildengine/testdata/projects/external/external.go diff --git a/buildengine/testdata/modules/external/ftl.toml b/buildengine/testdata/projects/external/ftl.toml similarity index 100% rename from buildengine/testdata/modules/external/ftl.toml rename to buildengine/testdata/projects/external/ftl.toml diff --git a/buildengine/testdata/modules/external/go.mod b/buildengine/testdata/projects/external/go.mod similarity index 100% rename from buildengine/testdata/modules/external/go.mod rename to buildengine/testdata/projects/external/go.mod diff --git a/buildengine/testdata/modules/external/go.sum b/buildengine/testdata/projects/external/go.sum similarity index 100% rename from buildengine/testdata/modules/external/go.sum rename to buildengine/testdata/projects/external/go.sum diff --git a/buildengine/testdata/modules/externalkotlin/ftl.toml b/buildengine/testdata/projects/externalkotlin/ftl.toml similarity index 100% rename from buildengine/testdata/modules/externalkotlin/ftl.toml rename to buildengine/testdata/projects/externalkotlin/ftl.toml diff --git a/buildengine/testdata/modules/externalkotlin/pom.xml b/buildengine/testdata/projects/externalkotlin/pom.xml similarity index 100% rename from buildengine/testdata/modules/externalkotlin/pom.xml rename to buildengine/testdata/projects/externalkotlin/pom.xml diff --git a/buildengine/testdata/modules/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt b/buildengine/testdata/projects/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt similarity index 100% rename from buildengine/testdata/modules/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt rename to buildengine/testdata/projects/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt diff --git a/buildengine/testdata/projects/lib/go.mod b/buildengine/testdata/projects/lib/go.mod new file mode 100644 index 0000000000..fda5d0e1f5 --- /dev/null +++ b/buildengine/testdata/projects/lib/go.mod @@ -0,0 +1,5 @@ +module lib + +go 1.22.0 + +replace github.com/TBD54566975/ftl => ../../.. diff --git a/buildengine/testdata/projects/lib/lib.go b/buildengine/testdata/projects/lib/lib.go new file mode 100644 index 0000000000..d4df247e2d --- /dev/null +++ b/buildengine/testdata/projects/lib/lib.go @@ -0,0 +1,11 @@ +package lib + +import ( + "context" + "fmt" + "ftl/alpha" +) + +func CreateEchoResponse(ctx context.Context, req alpha.EchoRequest) alpha.EchoResponse { + return alpha.EchoResponse{Message: fmt.Sprintf("Hello, %s!!!", req.Name)}, nil +} diff --git a/buildengine/testdata/projects/libkotlin/pom.xml b/buildengine/testdata/projects/libkotlin/pom.xml new file mode 100644 index 0000000000..79997b0ce0 --- /dev/null +++ b/buildengine/testdata/projects/libkotlin/pom.xml @@ -0,0 +1,153 @@ + + + 4.0.0 + + ftl + lib + 1.0-SNAPSHOT + + + 1.0-SNAPSHOT + 1.8 + 1.9.22 + true + ${java.version} + ${java.version} + + + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + + + xyz.block + ftl-runtime + ${ftl.version} + + + org.postgresql + postgresql + 42.7.2 + + + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + compile + + compile + + + + ${project.basedir}/src/main/kotlin + + + + + test-compile + + test-compile + + + + ${project.basedir}/src/test/kotlin + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.1 + + + + copy-dependencies + compile + + copy-dependencies + + + ${project.build.directory}/dependency + runtime + + + + + build-classpath + compile + + build-classpath + + + ${project.build.directory}/classpath.txt + dependency + + + + build-classpath-property + compile + + build-classpath + + + generated.classpath + ${project.build.directory}/dependency + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.5.0 + + + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/ftl + + + + + + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.codehaus.mojo + build-helper-maven-plugin + + + + com.github.ozsie + detekt-maven-plugin + + + + \ No newline at end of file diff --git a/buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt b/buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt new file mode 100644 index 0000000000..5bead81853 --- /dev/null +++ b/buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt @@ -0,0 +1,11 @@ +package lib + +import ftl.builtin.Empty +import ftl.echo.EchoRequest +import xyz.block.ftl.Context +import xyz.block.ftl.Verb + + +fun generateEchoText(req: EchoRequest): String { + return "Hello" +} diff --git a/buildengine/testdata/modules/other/ftl.toml b/buildengine/testdata/projects/other/ftl.toml similarity index 100% rename from buildengine/testdata/modules/other/ftl.toml rename to buildengine/testdata/projects/other/ftl.toml diff --git a/buildengine/testdata/modules/other/go.mod b/buildengine/testdata/projects/other/go.mod similarity index 100% rename from buildengine/testdata/modules/other/go.mod rename to buildengine/testdata/projects/other/go.mod diff --git a/buildengine/testdata/modules/other/go.sum b/buildengine/testdata/projects/other/go.sum similarity index 100% rename from buildengine/testdata/modules/other/go.sum rename to buildengine/testdata/projects/other/go.sum diff --git a/buildengine/testdata/modules/other/other.go b/buildengine/testdata/projects/other/other.go similarity index 100% rename from buildengine/testdata/modules/other/other.go rename to buildengine/testdata/projects/other/other.go diff --git a/buildengine/watch.go b/buildengine/watch.go index ada9a4f176..8bb628d9be 100644 --- a/buildengine/watch.go +++ b/buildengine/watch.go @@ -6,42 +6,41 @@ import ( "github.com/alecthomas/types/pubsub" - "github.com/TBD54566975/ftl/common/moduleconfig" "github.com/TBD54566975/ftl/internal/log" "github.com/TBD54566975/ftl/internal/maps" ) -// A WatchEvent is an event that occurs when a module is added, removed, or +// A WatchEvent is an event that occurs when a project is added, removed, or // changed. type WatchEvent interface{ watchEvent() } -type WatchEventModuleAdded struct{ Module Module } +type WatchEventProjectAdded struct{ Project Project } -func (WatchEventModuleAdded) watchEvent() {} +func (WatchEventProjectAdded) watchEvent() {} -type WatchEventModuleRemoved struct{ Module Module } +type WatchEventProjectRemoved struct{ Project Project } -func (WatchEventModuleRemoved) watchEvent() {} +func (WatchEventProjectRemoved) watchEvent() {} -type WatchEventModuleChanged struct { - Module Module - Change FileChangeType - Path string +type WatchEventProjectChanged struct { + Project Project + Change FileChangeType + Path string } -func (WatchEventModuleChanged) watchEvent() {} +func (WatchEventProjectChanged) watchEvent() {} -// Watch the given directories for new modules, deleted modules, and changes to -// existing modules, publishing a change event for each. -func Watch(ctx context.Context, period time.Duration, dirs ...string) *pubsub.Topic[WatchEvent] { +// Watch the given directories for new projects, deleted projects, and changes to +// existing projects, publishing a change event for each. +func Watch(ctx context.Context, period time.Duration, moduleDirs []string, externalLibDirs []string) *pubsub.Topic[WatchEvent] { logger := log.FromContext(ctx) topic := pubsub.New[WatchEvent]() go func() { - type moduleHashes struct { - Hashes FileHashes - Module Module + type projectHashes struct { + Hashes FileHashes + Project Project } - existingModules := map[string]moduleHashes{} + existingProjects := map[string]projectHashes{} wait := topic.Wait() for { select { @@ -55,52 +54,45 @@ func Watch(ctx context.Context, period time.Duration, dirs ...string) *pubsub.To return } - // Find all modules in the given directories. - moduleConfigs, err := DiscoverModules(ctx, dirs...) - if err != nil { - logger.Tracef("error discovering modules: %v", err) - continue - } - moduleConfigsByDir := maps.FromSlice(moduleConfigs, func(config moduleconfig.ModuleConfig) (string, moduleconfig.ModuleConfig) { - return config.Module, config + projects, _ := DiscoverProjects(ctx, moduleDirs, externalLibDirs, false) + + projectsByDir := maps.FromSlice(projects, func(project Project) (string, Project) { + return project.Config().Dir, project }) - // Trigger events for removed modules. - for _, existingModule := range existingModules { - if _, haveModule := moduleConfigsByDir[existingModule.Module.Module]; !haveModule { - logger.Debugf("module %s removed: %s", existingModule.Module.Module, existingModule.Module.Dir) - topic.Publish(WatchEventModuleRemoved{Module: existingModule.Module}) - delete(existingModules, existingModule.Module.ModuleConfig.Dir) + // Trigger events for removed projects. + for _, existingProject := range existingProjects { + existingConfig := existingProject.Project.Config() + if _, haveProject := projectsByDir[existingConfig.Dir]; !haveProject { + logger.Debugf("removed %s", existingProject.Project) + topic.Publish(WatchEventProjectRemoved{Project: existingProject.Project}) + delete(existingProjects, existingConfig.Dir) } } - // Compare the modules to the existing modules. - for _, config := range moduleConfigs { - existingModule, haveExistingModule := existingModules[config.Dir] - hashes, err := ComputeFileHashes(config) + // Compare the projects to the existing projects. + for _, project := range projectsByDir { + config := project.Config() + existingProject, haveExistingProject := existingProjects[config.Dir] + hashes, err := ComputeFileHashes(project) if err != nil { logger.Tracef("error computing file hashes for %s: %v", config.Dir, err) continue } - if haveExistingModule { - changeType, path, equal := CompareFileHashes(existingModule.Hashes, hashes) + if haveExistingProject { + changeType, path, equal := CompareFileHashes(existingProject.Hashes, hashes) if equal { continue } - logger.Debugf("module %s changed: %c%s", existingModule.Module.Module, changeType, path) - topic.Publish(WatchEventModuleChanged{Module: existingModule.Module, Change: changeType, Path: path}) - existingModules[config.Dir] = moduleHashes{Hashes: hashes, Module: existingModule.Module} - continue - } - - module, err := UpdateDependencies(ctx, config) - if err != nil { + logger.Debugf("changed %s: %c%s", project, changeType, path) + topic.Publish(WatchEventProjectChanged{Project: existingProject.Project, Change: changeType, Path: path}) + existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: existingProject.Project} continue } - logger.Debugf("module %s added: %s", module.Module, module.Dir) - topic.Publish(WatchEventModuleAdded{Module: module}) - existingModules[config.Dir] = moduleHashes{Hashes: hashes, Module: module} + logger.Debugf("added %s", project) + topic.Publish(WatchEventProjectAdded{Project: project}) + existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: project} } } }() diff --git a/buildengine/watch_test.go b/buildengine/watch_test.go index 9ab5e6286a..da30f8a5c8 100644 --- a/buildengine/watch_test.go +++ b/buildengine/watch_test.go @@ -21,7 +21,7 @@ func TestWatch(t *testing.T) { // Start the watch events := make(chan WatchEvent, 128) - watch := Watch(ctx, time.Millisecond*500, dir) + watch := Watch(ctx, time.Millisecond*500, []string{dir}, nil) watch.Subscribe(events) // Initiate a bunch of changes. @@ -57,18 +57,18 @@ func TestWatch(t *testing.T) { found := 0 for _, event := range allEvents { switch event := event.(type) { - case WatchEventModuleAdded: - if event.Module.Module == "one" || event.Module.Module == "two" { + case WatchEventProjectAdded: + if event.Project.Config().Key == "one" || event.Project.Config().Key == "two" { found++ } - case WatchEventModuleRemoved: - if event.Module.Module == "two" { + case WatchEventProjectRemoved: + if event.Project.Config().Key == "two" { found++ } - case WatchEventModuleChanged: - if event.Module.Module == "one" { + case WatchEventProjectChanged: + if event.Project.Config().Key == "one" { found++ } } diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index 312df166cb..d654b88a5c 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -11,11 +11,12 @@ import ( type buildCmd struct { Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` Dirs []string `arg:"" help:"Base directories containing modules." type:"existingdir" required:""` + External []string `help:"Directories for libraries that require FTL module stubs." type:"existingdir" optional:""` } func (b *buildCmd) Run(ctx context.Context) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, b.Dirs, b.External, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index 8ea47241c3..65ff8bb86d 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -17,7 +17,7 @@ type deployCmd struct { func (d *deployCmd) Run(ctx context.Context) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, d.Dirs, []string{}, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go index 024c0d1c41..68552c3482 100644 --- a/cmd/ftl/cmd_dev.go +++ b/cmd/ftl/cmd_dev.go @@ -15,6 +15,7 @@ import ( type devCmd struct { Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` Dirs []string `arg:"" help:"Base directories containing modules." type:"existingdir" required:""` + External []string `help:"Directories for libraries that require FTL module stubs." type:"existingdir" optional:""` Watch time.Duration `help:"Watch template directory at this frequency and regenerate on change." default:"500ms"` NoServe bool `help:"Do not start the FTL server." default:"false"` ServeCmd serveCmd `embed:""` @@ -48,7 +49,7 @@ func (d *devCmd) Run(ctx context.Context) error { } g.Go(func() error { - engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, d.Dirs, d.External, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 597a8639dc..342c0d3fca 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -25,7 +25,7 @@ import ( "github.com/TBD54566975/ftl/internal/log" ) -type externalModuleContext struct { +type ExternalModuleContext struct { ModuleDir string *schema.Schema GoVersion string @@ -48,7 +48,7 @@ type mainModuleContext struct { Replacements []*modfile.Replace } -func (b externalModuleContext) NonMainModules() []*schema.Module { +func (b ExternalModuleContext) NonMainModules() []*schema.Module { modules := make([]*schema.Module, 0, len(b.Modules)) for _, module := range b.Modules { if module.Name == b.Main { @@ -61,6 +61,10 @@ func (b externalModuleContext) NonMainModules() []*schema.Module { const buildDirName = "_ftl" +func buildDir(moduleDir string) string { + return filepath.Join(moduleDir, buildDirName) +} + // Build the given module. func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { replacements, goModVersion, err := updateGoModule(filepath.Join(moduleDir, "go.mod")) @@ -81,21 +85,16 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { funcs := maps.Clone(scaffoldFuncs) - buildDir := filepath.Join(moduleDir, buildDirName) - - // Wipe the modules directory to ensure we don't have any stale modules. - _ = os.RemoveAll(filepath.Join(buildDir, "go", "modules")) - logger.Debugf("Generating external modules") - if err := internal.ScaffoldZip(externalModuleTemplateFiles(), moduleDir, externalModuleContext{ + if err := generateExternalModules(ExternalModuleContext{ ModuleDir: moduleDir, GoVersion: goModVersion, FTLVersion: ftlVersion, Schema: sch, Main: config.Module, Replacements: replacements, - }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { - return err + }); err != nil { + return fmt.Errorf("failed to generate external modules: %w", err) } logger.Debugf("Extracting schema") @@ -107,6 +106,7 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { if err != nil { return fmt.Errorf("failed to marshal schema: %w", err) } + buildDir := buildDir(moduleDir) err = os.WriteFile(filepath.Join(buildDir, "schema.pb"), schemaBytes, 0600) if err != nil { return fmt.Errorf("failed to write schema: %w", err) @@ -172,6 +172,38 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error { return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } +func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error { + goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod")) + if err != nil { + return fmt.Errorf("failed to propagate replacements for library %s: %w", dir, err) + } + + ftlVersion := "" + if ftl.IsRelease(ftl.Version) { + ftlVersion = ftl.Version + } + + return generateExternalModules(ExternalModuleContext{ + ModuleDir: dir, + GoVersion: goModFile.Go.Version, + FTLVersion: ftlVersion, + Schema: schema, + Replacements: replacements, + }) + +} + +func generateExternalModules(context ExternalModuleContext) error { + // Wipe the modules directory to ensure we don't have any stale modules. + err := os.RemoveAll(filepath.Join(buildDir(context.ModuleDir), "go", "modules")) + if err != nil { + return err + } + + funcs := maps.Clone(scaffoldFuncs) + return internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) +} + func online() bool { _, err := net.LookupHost("proxy.golang.org") return err == nil @@ -286,25 +318,9 @@ func genType(module *schema.Module, t schema.Type) string { // Update go.mod file to include the FTL version and return the Go version and any replace directives. func updateGoModule(goModPath string) (replacements []*modfile.Replace, goVersion string, err error) { - goModBytes, err := os.ReadFile(goModPath) + goModFile, replacements, err := goModFileWithReplacements(goModPath) if err != nil { - return nil, "", fmt.Errorf("failed to read %s: %w", goModPath, err) - } - goModFile, err := modfile.Parse(goModPath, goModBytes, nil) - if err != nil { - return nil, "", fmt.Errorf("failed to parse %s: %w", goModPath, err) - } - - // Propagate any replace directives. - replacements = reflect.DeepCopy(goModFile.Replace) - for i, r := range replacements { - if strings.HasPrefix(r.New.Path, ".") { - abs, err := filepath.Abs(filepath.Join(filepath.Dir(goModPath), r.New.Path)) - if err != nil { - return nil, "", err - } - replacements[i].New.Path = abs - } + return nil, "", fmt.Errorf("failed to update %s: %w", goModPath, err) } // Early return if we're not updating anything. @@ -323,7 +339,7 @@ func updateGoModule(goModPath string) (replacements []*modfile.Replace, goVersio } defer os.Remove(tmpFile.Name()) // Delete the temp file if we error. defer tmpFile.Close() - goModBytes = modfile.Format(goModFile.Syntax) + goModBytes := modfile.Format(goModFile.Syntax) if _, err := tmpFile.Write(goModBytes); err != nil { return nil, "", fmt.Errorf("update %s: %w", goModPath, err) } @@ -333,6 +349,29 @@ func updateGoModule(goModPath string) (replacements []*modfile.Replace, goVersio return replacements, goModFile.Go.Version, nil } +func goModFileWithReplacements(goModPath string) (*modfile.File, []*modfile.Replace, error) { + goModBytes, err := os.ReadFile(goModPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read %s: %w", goModPath, err) + } + goModFile, err := modfile.Parse(goModPath, goModBytes, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse %s: %w", goModPath, err) + } + + replacements := reflect.DeepCopy(goModFile.Replace) + for i, r := range replacements { + if strings.HasPrefix(r.New.Path, ".") { + abs, err := filepath.Abs(filepath.Join(filepath.Dir(goModPath), r.New.Path)) + if err != nil { + return nil, nil, err + } + replacements[i].New.Path = abs + } + } + return goModFile, replacements, nil +} + func shouldUpdateVersion(goModfile *modfile.File) bool { for _, require := range goModfile.Require { if require.Mod.Path == "github.com/TBD54566975/ftl" && require.Mod.Version == ftl.Version {