From d5225267c3cc1e487d3f5324c99d587a63e6cf3c Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 18 Jun 2024 09:16:13 -0700 Subject: [PATCH] fix: remove unused external libs and buildengine.project --- buildengine/build.go | 59 +--- buildengine/build_go.go | 11 +- buildengine/build_go_test.go | 12 +- buildengine/build_kotlin.go | 48 ++-- buildengine/build_kotlin_test.go | 16 +- buildengine/deploy.go | 10 +- buildengine/deploy_test.go | 2 +- buildengine/deps.go | 35 ++- buildengine/deps_test.go | 16 +- buildengine/discover.go | 24 +- buildengine/discover_test.go | 77 +++--- buildengine/engine.go | 255 +++++++++--------- buildengine/engine_test.go | 4 +- buildengine/filehash.go | 4 +- buildengine/module.go | 31 +++ buildengine/project.go | 156 ----------- .../testdata/{projects => }/alpha/alpha.go | 0 .../testdata/{projects => }/alpha/ftl.toml | 0 .../testdata/{projects => }/alpha/go.mod | 2 +- .../testdata/{projects => }/alpha/go.sum | 0 .../testdata/{projects => }/alpha/pkg/pkg.go | 0 .../src/main/kotlin/ftl/alpha/Alpha.kt | 0 .../{projects => }/another/another.go | 0 .../testdata/{projects => }/another/ftl.toml | 0 .../testdata/{projects => }/another/go.mod | 2 +- .../testdata/{projects => }/another/go.sum | 0 .../{projects => }/depcycle1/depcycle1.go | 0 .../{projects => }/depcycle1/ftl.toml | 0 .../testdata/{projects => }/depcycle1/go.mod | 0 .../{projects => }/depcycle2/depcycle2.go | 0 .../{projects => }/depcycle2/ftl.toml | 0 .../testdata/{projects => }/depcycle2/go.mod | 0 .../{projects => }/echokotlin/ftl.toml | 0 .../{projects => }/echokotlin/pom.xml | 0 .../src/main/kotlin/ftl/echo/Echo.kt | 0 .../{projects => }/external/external.go | 0 .../testdata/{projects => }/external/ftl.toml | 0 .../{projects/lib => external}/go.mod | 2 +- .../testdata/{projects => }/external/go.sum | 0 .../{projects => }/externalkotlin/ftl.toml | 0 .../{projects => }/externalkotlin/pom.xml | 0 .../ftl/externalkotlin/ExternalKotlin.kt | 0 .../{projects => }/highgoversion/ftl.toml | 0 .../{projects => }/highgoversion/go.mod | 2 +- .../{projects => }/highgoversion/go.sum | 0 .../highgoversion/highgoversion.go | 0 .../testdata/{projects => }/other/ftl.toml | 0 .../testdata/{projects => }/other/go.mod | 2 +- .../testdata/{projects => }/other/go.sum | 0 .../testdata/{projects => }/other/other.go | 0 buildengine/testdata/projects/external/go.mod | 5 - buildengine/testdata/projects/lib/lib.go | 11 - .../testdata/projects/libkotlin/pom.xml | 153 ----------- .../libkotlin/src/main/kotlin/lib/Lib.kt | 11 - buildengine/watch.go | 106 ++++---- buildengine/watch_test.go | 30 +-- cmd/ftl/cmd_build.go | 8 +- cmd/ftl/cmd_deploy.go | 2 +- cmd/ftl/cmd_dev.go | 12 +- common/projectconfig/projectconfig.go | 1 - common/projectconfig/projectconfig_test.go | 3 +- .../projectconfig/testdata/ftl-project.toml | 1 - .../testdata/withMinVersion/ftl-project.toml | 1 - ftl-project.toml | 1 - 64 files changed, 348 insertions(+), 767 deletions(-) create mode 100644 buildengine/module.go delete mode 100644 buildengine/project.go rename buildengine/testdata/{projects => }/alpha/alpha.go (100%) rename buildengine/testdata/{projects => }/alpha/ftl.toml (100%) rename buildengine/testdata/{projects => }/alpha/go.mod (97%) rename buildengine/testdata/{projects => }/alpha/go.sum (100%) rename buildengine/testdata/{projects => }/alpha/pkg/pkg.go (100%) rename buildengine/testdata/{projects => }/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt (100%) rename buildengine/testdata/{projects => }/another/another.go (100%) rename buildengine/testdata/{projects => }/another/ftl.toml (100%) rename buildengine/testdata/{projects => }/another/go.mod (97%) rename buildengine/testdata/{projects => }/another/go.sum (100%) rename buildengine/testdata/{projects => }/depcycle1/depcycle1.go (100%) rename buildengine/testdata/{projects => }/depcycle1/ftl.toml (100%) rename buildengine/testdata/{projects => }/depcycle1/go.mod (100%) rename buildengine/testdata/{projects => }/depcycle2/depcycle2.go (100%) rename buildengine/testdata/{projects => }/depcycle2/ftl.toml (100%) rename buildengine/testdata/{projects => }/depcycle2/go.mod (100%) rename buildengine/testdata/{projects => }/echokotlin/ftl.toml (100%) rename buildengine/testdata/{projects => }/echokotlin/pom.xml (100%) rename buildengine/testdata/{projects => }/echokotlin/src/main/kotlin/ftl/echo/Echo.kt (100%) rename buildengine/testdata/{projects => }/external/external.go (100%) rename buildengine/testdata/{projects => }/external/ftl.toml (100%) rename buildengine/testdata/{projects/lib => external}/go.mod (74%) rename buildengine/testdata/{projects => }/external/go.sum (100%) rename buildengine/testdata/{projects => }/externalkotlin/ftl.toml (100%) rename buildengine/testdata/{projects => }/externalkotlin/pom.xml (100%) rename buildengine/testdata/{projects => }/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt (100%) rename buildengine/testdata/{projects => }/highgoversion/ftl.toml (100%) rename buildengine/testdata/{projects => }/highgoversion/go.mod (97%) rename buildengine/testdata/{projects => }/highgoversion/go.sum (100%) rename buildengine/testdata/{projects => }/highgoversion/highgoversion.go (100%) rename buildengine/testdata/{projects => }/other/ftl.toml (100%) rename buildengine/testdata/{projects => }/other/go.mod (97%) rename buildengine/testdata/{projects => }/other/go.sum (100%) rename buildengine/testdata/{projects => }/other/other.go (100%) delete mode 100644 buildengine/testdata/projects/external/go.mod delete mode 100644 buildengine/testdata/projects/lib/lib.go delete mode 100644 buildengine/testdata/projects/libkotlin/pom.xml delete mode 100644 buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt diff --git a/buildengine/build.go b/buildengine/build.go index f9213071ae..2df71fa7b9 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "google.golang.org/protobuf/proto" @@ -16,49 +15,39 @@ import ( "github.com/TBD54566975/ftl/internal/errors" "github.com/TBD54566975/ftl/internal/flock" "github.com/TBD54566975/ftl/internal/log" - "github.com/TBD54566975/ftl/internal/slices" ) const BuildLockTimeout = time.Minute -// Build a project in the given directory given the schema and project config. -// -// For a module, this will build the module. For an external library, this will build stubs for imported modules. +// Build a module in the given directory given the schema and module config. // // A lock file is used to ensure that only one build is running at a time. -func Build(ctx context.Context, sch *schema.Schema, project Project, filesTransaction ModifyFilesTransaction) error { - switch project := project.(type) { - case Module: - return buildModule(ctx, sch, project, filesTransaction) - case ExternalLibrary: - return buildExternalLibrary(ctx, sch, project) - default: - panic(fmt.Sprintf("unsupported project type: %T", project)) - } +func Build(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { + return buildModule(ctx, sch, module, filesTransaction) } func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTransaction ModifyFilesTransaction) error { - release, err := flock.Acquire(ctx, filepath.Join(module.Dir, ".ftl.lock"), BuildLockTimeout) + release, err := flock.Acquire(ctx, filepath.Join(module.Config.Dir, ".ftl.lock"), BuildLockTimeout) if err != nil { return err } defer release() //nolint:errcheck - logger := log.FromContext(ctx).Scope(module.Module) + logger := log.FromContext(ctx).Scope(module.Config.Module) ctx = log.ContextWithLogger(ctx, logger) // clear the deploy directory before extracting schema - if err := os.RemoveAll(module.AbsDeployDir()); err != nil { + if err := os.RemoveAll(module.Config.AbsDeployDir()); err != nil { return fmt.Errorf("failed to clear errors: %w", err) } logger.Infof("Building module") - switch module.Language { + switch module.Config.Language { case "go": err = buildGoModule(ctx, sch, module, filesTransaction) case "kotlin": err = buildKotlinModule(ctx, sch, module) default: - return fmt.Errorf("unknown language %q", module.Language) + return fmt.Errorf("unknown language %q", module.Config.Language) } var errs []error @@ -66,7 +55,7 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTr errs = append(errs, err) } // read runtime-specific build errors from the build directory - errorList, err := loadProtoErrors(module.ModuleConfig) + errorList, err := loadProtoErrors(module.Config) if err != nil { return fmt.Errorf("failed to read build errors for module: %w", err) } @@ -82,34 +71,8 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module, filesTr return nil } -func buildExternalLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error { - logger := log.FromContext(ctx).Scope(filepath.Base(lib.Dir)) - ctx = log.ContextWithLogger(ctx, logger) - - imported := slices.Map(sch.Modules, func(m *schema.Module) string { - return m.Name - }) - logger.Debugf("Generating stubs [%s] for %v", strings.Join(imported, ", "), lib) - - switch lib.Language { - case "go": - if err := buildGoLibrary(ctx, sch, lib); err != nil { - return err - } - case "kotlin": - if err := buildKotlinLibrary(ctx, sch, lib); err != nil { - return err - } - default: - return fmt.Errorf("unknown language %q for library %q", lib.Language, lib.Config().Key) - } - - logger.Infof("Generated stubs [%s] for %v", strings.Join(imported, ", "), lib) - return nil -} - -func loadProtoErrors(module moduleconfig.ModuleConfig) (*schema.ErrorList, error) { - f := filepath.Join(module.AbsDeployDir(), module.Errors) +func loadProtoErrors(config moduleconfig.ModuleConfig) (*schema.ErrorList, error) { + f := filepath.Join(config.AbsDeployDir(), config.Errors) if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) { return &schema.ErrorList{Errors: make([]*schema.Error, 0)}, nil } diff --git a/buildengine/build_go.go b/buildengine/build_go.go index ce7d6c6770..ac03a139a8 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -9,15 +9,8 @@ import ( ) func buildGoModule(ctx context.Context, sch *schema.Schema, module Module, transaction ModifyFilesTransaction) error { - if err := compile.Build(ctx, module.Dir, sch, transaction); err != nil { - return fmt.Errorf("failed to build module %q: %w", module.Config().Key, err) - } - return nil -} - -func buildGoLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error { - if err := compile.GenerateStubsForExternalLibrary(ctx, lib.Dir, sch); err != nil { - return fmt.Errorf("failed to generate stubs for library %q: %w", lib.Config().Key, err) + if err := compile.Build(ctx, module.Config.Dir, sch, transaction); err != nil { + return fmt.Errorf("failed to build module %q: %w", module.Config.Module, err) } return nil } diff --git a/buildengine/build_go_test.go b/buildengine/build_go_test.go index 60ff05cb04..498e26e43b 100644 --- a/buildengine/build_go_test.go +++ b/buildengine/build_go_test.go @@ -177,7 +177,7 @@ func init() { } ` bctx := buildContext{ - moduleDir: "testdata/projects/another", + moduleDir: "testdata/another", buildDir: "_ftl", sch: sch, } @@ -197,7 +197,7 @@ func TestGoBuildClearsBuildDir(t *testing.T) { }, } bctx := buildContext{ - moduleDir: "testdata/projects/another", + moduleDir: "testdata/another", buildDir: "_ftl", sch: sch, } @@ -255,7 +255,7 @@ func Call(context.Context, Req) (Resp, error) { } ` bctx := buildContext{ - moduleDir: "testdata/projects/another", + moduleDir: "testdata/another", buildDir: "_ftl", sch: sch, } @@ -269,7 +269,7 @@ func TestExternalType(t *testing.T) { t.SkipNow() } bctx := buildContext{ - moduleDir: "testdata/projects/external", + moduleDir: "testdata/external", buildDir: "_ftl", sch: &schema.Schema{}, } @@ -301,7 +301,7 @@ func TestGoModVersion(t *testing.T) { }, } bctx := buildContext{ - moduleDir: "testdata/projects/highgoversion", + moduleDir: "testdata/highgoversion", buildDir: "_ftl", sch: sch, } @@ -344,7 +344,7 @@ func TestGeneratedTypeRegistry(t *testing.T) { expected, err := os.ReadFile("testdata/type_registry_main.go") assert.NoError(t, err) bctx := buildContext{ - moduleDir: "testdata/projects/other", + moduleDir: "testdata/other", buildDir: "_ftl", sch: sch, } diff --git a/buildengine/build_kotlin.go b/buildengine/build_kotlin.go index 9ce1971bab..c50858808c 100644 --- a/buildengine/build_kotlin.go +++ b/buildengine/build_kotlin.go @@ -23,18 +23,14 @@ import ( ) type externalModuleContext struct { - project Project + module Module *schema.Schema } func (e externalModuleContext) ExternalModules() []*schema.Module { - name := "" - if module, ok := e.project.(Module); ok { - name = module.Module - } modules := make([]*schema.Module, 0, len(e.Modules)) for _, module := range e.Modules { - if module.Name == name { + if module.Name == e.module.Config.Module { continue } modules = append(modules, module) @@ -44,34 +40,27 @@ func (e externalModuleContext) ExternalModules() []*schema.Module { func buildKotlinModule(ctx context.Context, sch *schema.Schema, module Module) error { logger := log.FromContext(ctx) - if err := SetPOMProperties(ctx, module.Dir); err != nil { - return fmt.Errorf("unable to update ftl.version in %s: %w", module.Dir, err) + if err := SetPOMProperties(ctx, module.Config.Dir); err != nil { + return fmt.Errorf("unable to update ftl.version in %s: %w", module.Config.Dir, err) } - if err := generateExternalModules(ctx, &module, sch); err != nil { - return fmt.Errorf("unable to generate external modules for %s: %w", module.Module, err) + if err := generateExternalModules(ctx, module, sch); err != nil { + return fmt.Errorf("unable to generate external modules for %s: %w", module.Config.Module, err) } if err := prepareFTLRoot(module); err != nil { - return fmt.Errorf("unable to prepare FTL root for %s: %w", module.Module, err) + return fmt.Errorf("unable to prepare FTL root for %s: %w", module.Config.Module, err) } - logger.Debugf("Using build command '%s'", module.Build) - err := exec.Command(ctx, log.Debug, module.Dir, "bash", "-c", module.Build).RunBuffered(ctx) + logger.Debugf("Using build command '%s'", module.Config.Build) + err := exec.Command(ctx, log.Debug, module.Config.Dir, "bash", "-c", module.Config.Build).RunBuffered(ctx) if err != nil { - return fmt.Errorf("failed to build module %q: %w", module.Module, err) + return fmt.Errorf("failed to build module %q: %w", module.Config.Module, err) } return nil } -func buildKotlinLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error { - if err := generateExternalModules(ctx, &lib, sch); err != nil { - return fmt.Errorf("unable to generate external modules for %q: %w", lib.Config().Key, err) - } - return nil -} - // SetPOMProperties updates the ftl.version properties in the // pom.xml file in the given base directory. func SetPOMProperties(ctx context.Context, baseDir string) error { @@ -104,7 +93,7 @@ func SetPOMProperties(ctx context.Context, baseDir string) error { } func prepareFTLRoot(module Module) error { - buildDir := module.AbsDeployDir() + buildDir := module.Config.AbsDeployDir() if err := os.MkdirAll(buildDir, 0700); err != nil { return err } @@ -118,7 +107,7 @@ SchemaExtractorRuleSet: detektYmlPath := filepath.Join(buildDir, "detekt.yml") if err := os.WriteFile(detektYmlPath, []byte(fileContent), 0600); err != nil { - return fmt.Errorf("unable to configure detekt for %s: %w", module.Module, err) + return fmt.Errorf("unable to configure detekt for %s: %w", module.Config.Module, err) } mainFilePath := filepath.Join(buildDir, "main") @@ -127,23 +116,22 @@ SchemaExtractorRuleSet: exec java -cp "classes:$(cat classpath.txt)" xyz.block.ftl.main.MainKt ` if err := os.WriteFile(mainFilePath, []byte(mainFile), 0700); err != nil { //nolint:gosec - return fmt.Errorf("unable to configure main executable for %s: %w", module.Module, err) + return fmt.Errorf("unable to configure main executable for %s: %w", module.Config.Module, err) } return nil } -func generateExternalModules(ctx context.Context, project Project, sch *schema.Schema) error { +func generateExternalModules(ctx context.Context, module Module, sch *schema.Schema) error { logger := log.FromContext(ctx) funcs := maps.Clone(scaffoldFuncs) - config := project.Config() // Wipe the modules directory to ensure we don't have any stale modules. - _ = os.RemoveAll(filepath.Join(config.Dir, "target", "generated-sources", "ftl")) + _ = os.RemoveAll(filepath.Join(module.Config.Dir, "target", "generated-sources", "ftl")) logger.Debugf("Generating external modules") - return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), config.Dir, externalModuleContext{ - project: project, - Schema: sch, + return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), module.Config.Dir, externalModuleContext{ + module: module, + Schema: sch, }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) } diff --git a/buildengine/build_kotlin_test.go b/buildengine/build_kotlin_test.go index 83217582fb..3db650b2f5 100644 --- a/buildengine/build_kotlin_test.go +++ b/buildengine/build_kotlin_test.go @@ -23,7 +23,7 @@ package ftl.test ` bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -40,7 +40,7 @@ func TestKotlinBuildClearsBuildDir(t *testing.T) { }, } bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -165,7 +165,7 @@ data class TestResponse( ` bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -227,7 +227,7 @@ fun testVerb(context: Context, req: Request): ftl.builtin.Empty = throw NotImplementedError("Verb stubs should not be called directly, instead use context.call(::testVerb, ...)") ` bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -281,7 +281,7 @@ data class HttpResponse( class Empty ` bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -326,7 +326,7 @@ fun emptyVerb(context: Context, req: ftl.builtin.Empty): ftl.builtin.Empty = thr NotImplementedError("Verb stubs should not be called directly, instead use context.call(::emptyVerb, ...)") ` bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -407,7 +407,7 @@ fun nothing(context: Context): Unit = throw NotImplementedError("Verb stubs should not be called directly, instead use context.callEmpty(::nothing, ...)") ` bctx := buildContext{ - moduleDir: "testdata/projects/echokotlin", + moduleDir: "testdata/echokotlin", buildDir: "target", sch: sch, } @@ -421,7 +421,7 @@ func TestKotlinExternalType(t *testing.T) { t.SkipNow() } bctx := buildContext{ - moduleDir: "testdata/projects/externalkotlin", + moduleDir: "testdata/externalkotlin", buildDir: "target", sch: &schema.Schema{}, } diff --git a/buildengine/deploy.go b/buildengine/deploy.go index 725e13bf3c..983d8aea74 100644 --- a/buildengine/deploy.go +++ b/buildengine/deploy.go @@ -36,12 +36,12 @@ type DeployClient interface { // Deploy a module to the FTL controller with the given number of replicas. Optionally wait for the deployment to become ready. func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnline bool, client DeployClient) error { - logger := log.FromContext(ctx).Scope(module.Module) + logger := log.FromContext(ctx).Scope(module.Config.Module) ctx = log.ContextWithLogger(ctx, logger) logger.Infof("Deploying module") - deployDir := module.AbsDeployDir() - files, err := findFiles(deployDir, module.Deploy) + deployDir := module.Config.AbsDeployDir() + files, err := findFiles(deployDir, module.Config.Deploy) if err != nil { logger.Errorf(err, "failed to find files in %s", deployDir) return err @@ -57,9 +57,9 @@ func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnl return err } - moduleSchema, err := loadProtoSchema(deployDir, module.ModuleConfig, replicas) + moduleSchema, err := loadProtoSchema(deployDir, module.Config, replicas) if err != nil { - return fmt.Errorf("failed to load protobuf schema from %q: %w", module.Schema, err) + return fmt.Errorf("failed to load protobuf schema from %q: %w", module.Config.Schema, err) } logger.Debugf("Uploading %d/%d files", len(gadResp.Msg.MissingDigests), len(files)) diff --git a/buildengine/deploy_test.go b/buildengine/deploy_test.go index d2b92e529d..029f2d33af 100644 --- a/buildengine/deploy_test.go +++ b/buildengine/deploy_test.go @@ -67,7 +67,7 @@ func TestDeploy(t *testing.T) { } ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{})) - modulePath := "testdata/projects/another" + modulePath := "testdata/another" module, err := LoadModule(modulePath) assert.NoError(t, err) diff --git a/buildengine/deps.go b/buildengine/deps.go index ae7b7be3e0..5d0825cbca 100644 --- a/buildengine/deps.go +++ b/buildengine/deps.go @@ -19,14 +19,14 @@ import ( "github.com/TBD54566975/ftl/internal/log" ) -// UpdateDependencies finds the dependencies for a project and returns a -// Project with those dependencies populated. -func UpdateDependencies(ctx context.Context, project Project) (Project, error) { +// UpdateDependencies finds the dependencies for a module and returns a +// Module with those dependencies populated. +func UpdateDependencies(ctx context.Context, module Module) (Module, error) { logger := log.FromContext(ctx) - logger.Debugf("Extracting dependencies for %s %q", project.TypeString(), project.Config().Key) - dependencies, err := extractDependencies(project) + logger.Debugf("Extracting dependencies for %q", module.Config.Module) + dependencies, err := extractDependencies(module) if err != nil { - return Project(&Module{}), err + return Module{}, err } containsBuiltin := false for _, dep := range dependencies { @@ -39,29 +39,24 @@ func UpdateDependencies(ctx context.Context, project Project) (Project, error) { dependencies = append(dependencies, "builtin") } - out := project.CopyWithDependencies(dependencies) + out := module.CopyWithDependencies(dependencies) return out, nil } -func extractDependencies(project Project) ([]string, error) { - config := project.Config() - name := "" - if config, ok := project.(Module); ok { - name = config.Module - } - switch config.Language { +func extractDependencies(module Module) ([]string, error) { + switch module.Config.Language { case "go": - return extractGoFTLImports(name, config.Dir) + return extractGoFTLImports(module.Config.Module, module.Config.Dir) case "kotlin": - return extractKotlinFTLImports(name, config.Dir) + return extractKotlinFTLImports(module.Config.Module, module.Config.Dir) default: - return nil, fmt.Errorf("unsupported language: %s", config.Language) + return nil, fmt.Errorf("unsupported language: %s", module.Config.Language) } } -func extractGoFTLImports(self, dir string) ([]string, error) { +func extractGoFTLImports(moduleName, dir string) ([]string, error) { dependencies := map[string]bool{} fset := token.NewFileSet() err := WalkDir(dir, func(path string, d fs.DirEntry) error { @@ -86,7 +81,7 @@ func extractGoFTLImports(self, dir string) ([]string, error) { continue } module := strings.Split(strings.TrimPrefix(path, "ftl/"), "/")[0] - if module == self { + if module == moduleName { continue } dependencies[module] = true @@ -96,7 +91,7 @@ func extractGoFTLImports(self, dir string) ([]string, error) { return nil }) if err != nil { - return nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", self, err) + return nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", moduleName, err) } modules := maps.Keys(dependencies) sort.Strings(modules) diff --git a/buildengine/deps_test.go b/buildengine/deps_test.go index 3e558521f7..834300b93a 100644 --- a/buildengine/deps_test.go +++ b/buildengine/deps_test.go @@ -7,25 +7,13 @@ import ( ) func TestExtractModuleDepsGo(t *testing.T) { - deps, err := extractGoFTLImports("test", "testdata/projects/alpha") + deps, err := extractGoFTLImports("test", "testdata/alpha") assert.NoError(t, err) assert.Equal(t, []string{"another", "other"}, deps) } func TestExtractModuleDepsKotlin(t *testing.T) { - deps, err := extractKotlinFTLImports("test", "testdata/projects/alphakotlin") + deps, err := extractKotlinFTLImports("test", "testdata/alphakotlin") assert.NoError(t, err) assert.Equal(t, []string{"builtin", "other"}, deps) } - -func TestExtractLibraryDepsGo(t *testing.T) { - deps, err := extractGoFTLImports("test", "testdata/projects/lib") - assert.NoError(t, err) - assert.Equal(t, []string{"alpha"}, deps) -} - -func TestExtractLibraryDepsKotlin(t *testing.T) { - deps, err := extractKotlinFTLImports("test", "testdata/projects/libkotlin") - assert.NoError(t, err) - assert.Equal(t, []string{"builtin", "echo"}, deps) -} diff --git a/buildengine/discover.go b/buildengine/discover.go index 844db4c941..cec074373a 100644 --- a/buildengine/discover.go +++ b/buildengine/discover.go @@ -10,11 +10,10 @@ import ( "github.com/TBD54566975/ftl/internal/log" ) -// DiscoverProjects recursively loads all modules under the given directories -// (or if none provided, the current working directory is used) and external -// libraries in externalLibDirs. -func DiscoverProjects(ctx context.Context, moduleDirs []string, externalLibDirs []string) ([]Project, error) { - out := []Project{} +// DiscoverModules recursively loads all modules under the given directories +// (or if none provided, the current working directory is used). +func DiscoverModules(ctx context.Context, moduleDirs []string) ([]Module, error) { + out := []Module{} logger := log.FromContext(ctx) modules, err := discoverModules(moduleDirs...) @@ -22,17 +21,8 @@ func DiscoverProjects(ctx context.Context, moduleDirs []string, externalLibDirs logger.Tracef("error discovering modules: %v", err) return nil, err } - for _, module := range modules { - out = append(out, Project(module)) - } - for _, dir := range externalLibDirs { - lib, err := LoadExternalLibrary(dir) - if err != nil { - logger.Tracef("error discovering external library: %v", err) - return nil, err - } - out = append(out, Project(lib)) - } + + out = append(out, modules...) return out, nil } @@ -66,7 +56,7 @@ func discoverModules(dirs ...string) ([]Module, error) { } } sort.Slice(out, func(i, j int) bool { - return out[i].Config().Key < out[j].Config().Key + return out[i].Config.Module < out[j].Config.Module }) return out, nil } diff --git a/buildengine/discover_test.go b/buildengine/discover_test.go index a3d7d50ef5..b63d7d3416 100644 --- a/buildengine/discover_test.go +++ b/buildengine/discover_test.go @@ -12,12 +12,12 @@ import ( func TestDiscoverModules(t *testing.T) { ctx := log.ContextWithNewDefaultLogger(context.Background()) - projects, err := DiscoverProjects(ctx, []string{"testdata/projects"}, []string{"testdata/projects/lib", "testdata/projects/libkotlin"}) + modules, err := DiscoverModules(ctx, []string{"testdata"}) assert.NoError(t, err) - expected := []Project{ - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/alpha", + expected := []Module{ + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/alpha", Language: "go", Realm: "home", Module: "alpha", @@ -25,12 +25,12 @@ func TestDiscoverModules(t *testing.T) { DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../../go-runtime/ftl/**/*.go"}, + Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/another", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/another", Language: "go", Realm: "home", Module: "another", @@ -38,12 +38,12 @@ func TestDiscoverModules(t *testing.T) { DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../../go-runtime/ftl/**/*.go"}, + Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/depcycle1", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/depcycle1", Language: "go", Realm: "home", Module: "depcycle1", @@ -54,9 +54,9 @@ func TestDiscoverModules(t *testing.T) { Watch: []string{"**/*.go", "go.mod", "go.sum"}, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/depcycle2", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/depcycle2", Language: "go", Realm: "home", Module: "depcycle2", @@ -67,9 +67,9 @@ func TestDiscoverModules(t *testing.T) { Watch: []string{"**/*.go", "go.mod", "go.sum"}, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/echokotlin", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/echokotlin", Language: "kotlin", Realm: "home", Module: "echo", @@ -90,9 +90,9 @@ func TestDiscoverModules(t *testing.T) { }, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/external", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/external", Language: "go", Realm: "home", Module: "external", @@ -110,9 +110,9 @@ func TestDiscoverModules(t *testing.T) { }, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/externalkotlin", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/externalkotlin", Language: "kotlin", Realm: "home", Module: "externalkotlin", @@ -133,9 +133,9 @@ func TestDiscoverModules(t *testing.T) { }, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/highgoversion", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/highgoversion", Language: "go", Realm: "home", Module: "highgoversion", @@ -143,12 +143,12 @@ func TestDiscoverModules(t *testing.T) { DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../../go-runtime/ftl/**/*.go"}, + Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, }, }, - Module{ - ModuleConfig: moduleconfig.ModuleConfig{ - Dir: "testdata/projects/other", + { + Config: moduleconfig.ModuleConfig{ + Dir: "testdata/other", Language: "go", Realm: "home", Module: "other", @@ -156,17 +156,10 @@ func TestDiscoverModules(t *testing.T) { DeployDir: "_ftl", Schema: "schema.pb", Errors: "errors.pb", - Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../../go-runtime/ftl/**/*.go"}, + Watch: []string{"**/*.go", "go.mod", "go.sum", "../../../go-runtime/ftl/**/*.go"}, }, }, - ExternalLibrary{ - Dir: "testdata/projects/lib", - Language: "go", - }, - ExternalLibrary{ - Dir: "testdata/projects/libkotlin", - Language: "kotlin", - }, } - assert.Equal(t, expected, projects) + + assert.Equal(t, expected, modules) } diff --git a/buildengine/engine.go b/buildengine/engine.go index 831aa478c1..258219d390 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -32,32 +32,32 @@ type schemaChange struct { *schema.Module } -type projectMeta struct { - project Project +// moduleMeta is a wrapper around a module that includes the last build's start time. +type moduleMeta struct { + module Module lastBuildStartTime time.Time } type Listener interface { - OnBuildStarted(project Project) + OnBuildStarted(module Module) } -type BuildStartedListenerFunc func(project Project) +type BuildStartedListenerFunc func(module Module) -func (b BuildStartedListenerFunc) OnBuildStarted(project Project) { b(project) } +func (b BuildStartedListenerFunc) OnBuildStarted(module Module) { b(module) } // Engine for building a set of modules. type Engine struct { client ftlv1connect.ControllerServiceClient - projectMetas *xsync.MapOf[ProjectKey, projectMeta] + moduleMetas *xsync.MapOf[string, moduleMeta] moduleDirs []string - externalDirs []string watcher *Watcher controllerSchema *xsync.MapOf[string, *schema.Module] schemaChanges *pubsub.Topic[schemaChange] cancel func() parallelism int listener Listener - projectsToBuild *xsync.MapOf[ProjectKey, bool] + modulesToBuild *xsync.MapOf[string, bool] } type Option func(o *Engine) @@ -82,18 +82,17 @@ func WithListener(listener Listener) Option { // pull in missing schemas. // // "dirs" are directories to scan for local modules. -func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, externalDirs []string, options ...Option) (*Engine, error) { +func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, moduleDirs []string, options ...Option) (*Engine, error) { ctx = rpc.ContextWithClient(ctx, client) e := &Engine{ client: client, moduleDirs: moduleDirs, - externalDirs: externalDirs, - projectMetas: xsync.NewMapOf[ProjectKey, projectMeta](), + moduleMetas: xsync.NewMapOf[string, moduleMeta](), watcher: NewWatcher(), controllerSchema: xsync.NewMapOf[string, *schema.Module](), schemaChanges: pubsub.New[schemaChange](), parallelism: runtime.NumCPU(), - projectsToBuild: xsync.NewMapOf[ProjectKey, bool](), + modulesToBuild: xsync.NewMapOf[string, bool](), } for _, option := range options { option(e) @@ -102,17 +101,17 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, modul ctx, cancel := context.WithCancel(ctx) e.cancel = cancel - projects, err := DiscoverProjects(ctx, moduleDirs, externalDirs) + modules, err := DiscoverModules(ctx, moduleDirs) if err != nil { - return nil, fmt.Errorf("could not find projects: %w", err) + return nil, fmt.Errorf("could not find modules: %w", err) } - for _, project := range projects { - project, err = UpdateDependencies(ctx, project) + for _, module := range modules { + module, err = UpdateDependencies(ctx, module) if err != nil { return nil, err } - e.projectMetas.Store(project.Config().Key, projectMeta{project: project}) - e.projectsToBuild.Store(project.Config().Key, true) + e.moduleMetas.Store(module.Config.Module, moduleMeta{module: module}) + e.modulesToBuild.Store(module.Config.Module, true) } if client == nil { @@ -171,36 +170,36 @@ func (e *Engine) Close() error { // // If no modules are provided, the entire graph is returned. An error is returned if // any dependencies are missing. -func (e *Engine) Graph(projects ...ProjectKey) (map[string][]string, error) { +func (e *Engine) Graph(moduleNames ...string) (map[string][]string, error) { out := map[string][]string{} - if len(projects) == 0 { - e.projectMetas.Range(func(key ProjectKey, _ projectMeta) bool { - projects = append(projects, key) + if len(moduleNames) == 0 { + e.moduleMetas.Range(func(name string, _ moduleMeta) bool { + moduleNames = append(moduleNames, name) return true }) } - for _, key := range projects { - if err := e.buildGraph(string(key), out); err != nil { + for _, name := range moduleNames { + if err := e.buildGraph(name, out); err != nil { return nil, err } } return out, nil } -func (e *Engine) buildGraph(key string, out map[string][]string) error { +func (e *Engine) buildGraph(moduleName string, out map[string][]string) error { var deps []string // Short-circuit previously explored nodes - if _, ok := out[key]; ok { + if _, ok := out[moduleName]; ok { return nil } - if meta, ok := e.projectMetas.Load(ProjectKey(key)); ok { - deps = meta.project.Config().Dependencies - } else if sch, ok := e.controllerSchema.Load(key); ok { + if meta, ok := e.moduleMetas.Load(moduleName); ok { + deps = meta.module.Dependencies + } else if sch, ok := e.controllerSchema.Load(moduleName); ok { deps = sch.Imports() } else { - return fmt.Errorf("module %q not found", key) + return fmt.Errorf("module %q not found", moduleName) } - out[key] = deps + out[moduleName] = deps for _, dep := range deps { if err := e.buildGraph(dep, out); err != nil { return err @@ -251,7 +250,7 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration }() watchEvents := make(chan WatchEvent, 128) - topic, err := e.watcher.Watch(ctx, period, e.moduleDirs, e.externalDirs) + topic, err := e.watcher.Watch(ctx, period, e.moduleDirs) if err != nil { return err } @@ -296,34 +295,34 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration didUpdateDeployments = false case event := <-watchEvents: switch event := event.(type) { - case WatchEventProjectAdded: - config := event.Project.Config() - if _, exists := e.projectMetas.Load(config.Key); !exists { - e.projectMetas.Store(config.Key, projectMeta{project: event.Project}) - err := e.buildAndDeploy(ctx, 1, true, config.Key) + case WatchEventModuleAdded: + config := event.Module.Config + if _, exists := e.moduleMetas.Load(config.Module); !exists { + e.moduleMetas.Store(config.Module, moduleMeta{module: event.Module}) + err := e.buildAndDeploy(ctx, 1, true, config.Module) if err != nil { - logger.Errorf(err, "deploy %s failed", config.Key) + logger.Errorf(err, "deploy %s failed", config.Module) } else { didUpdateDeployments = true } } - case WatchEventProjectRemoved: - config := event.Project.Config() - if module, ok := event.Project.(Module); ok { - err := teminateModuleDeployment(ctx, e.client, module.Module) - if err != nil { - logger.Errorf(err, "terminate %s failed", module.Module) - } else { - didUpdateDeployments = true - } + case WatchEventModuleRemoved: + config := event.Module.Config + + err := teminateModuleDeployment(ctx, e.client, config.Module) + if err != nil { + logger.Errorf(err, "terminate %s failed", config.Module) + } else { + didUpdateDeployments = true } - e.projectMetas.Delete(config.Key) - case WatchEventProjectChanged: - config := event.Project.Config() - meta, ok := e.projectMetas.Load(config.Key) + e.moduleMetas.Delete(config.Module) + case WatchEventModuleChanged: + config := event.Module.Config + + meta, ok := e.moduleMetas.Load(config.Module) if !ok { - logger.Warnf("project %q not found", config.Key) + logger.Warnf("module %q not found", config.Module) continue } @@ -331,14 +330,9 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration logger.Debugf("Skipping build and deploy; event time %v is before the last build time %v", event.Time, meta.lastBuildStartTime) continue // Skip this event as it's outdated } - err := e.buildAndDeploy(ctx, 1, true, config.Key) + err := e.buildAndDeploy(ctx, 1, true, config.Module) if err != nil { - switch project := event.Project.(type) { - case Module: - logger.Errorf(err, "build and deploy failed for module %q", project.Config().Key) - case ExternalLibrary: - logger.Errorf(err, "build failed for library %q: %v", project.Config().Key, err) - } + logger.Errorf(err, "build and deploy failed for module %q", event.Module.Config.Module) } else { didUpdateDeployments = true } @@ -361,11 +355,10 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration moduleHashes[change.Name] = hash - dependentProjectKeys := e.getDependentProjectKeys(change.Name) - if len(dependentProjectKeys) > 0 { - //TODO: inaccurate log message for ext libs - logger.Infof("%s's schema changed; processing %s", change.Name, strings.Join(StringsFromProjectKeys(dependentProjectKeys), ", ")) - err = e.buildAndDeploy(ctx, 1, true, dependentProjectKeys...) + dependentModuleNames := e.getDependentModuleNames(change.Name) + if len(dependentModuleNames) > 0 { + logger.Infof("%s's schema changed; processing %s", change.Name, strings.Join(dependentModuleNames, ", ")) + err = e.buildAndDeploy(ctx, 1, true, dependentModuleNames...) if err != nil { logger.Errorf(err, "deploy %s failed", change.Name) } else { @@ -386,24 +379,24 @@ func computeModuleHash(module *schema.Module) ([]byte, error) { return hasher.Sum(nil), nil } -func (e *Engine) getDependentProjectKeys(name string) []ProjectKey { - dependentProjectKeys := map[ProjectKey]bool{} - e.projectMetas.Range(func(key ProjectKey, meta projectMeta) bool { - for _, dep := range meta.project.Config().Dependencies { - if dep == name { - dependentProjectKeys[key] = true +func (e *Engine) getDependentModuleNames(moduleName string) []string { + dependentModuleNames := map[string]bool{} + e.moduleMetas.Range(func(name string, meta moduleMeta) bool { + for _, dep := range meta.module.Dependencies { + if dep == moduleName { + dependentModuleNames[name] = true } } return true }) - return maps.Keys(dependentProjectKeys) + return maps.Keys(dependentModuleNames) } -func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDeployOnline bool, projects ...ProjectKey) error { +func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDeployOnline bool, moduleNames ...string) error { logger := log.FromContext(ctx) - if len(projects) == 0 { - e.projectMetas.Range(func(key ProjectKey, meta projectMeta) bool { - projects = append(projects, key) + if len(moduleNames) == 0 { + e.moduleMetas.Range(func(name string, meta moduleMeta) bool { + moduleNames = append(moduleNames, name) return true }) } @@ -412,18 +405,13 @@ func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDepl deployGroup := errgroup.Group{} buildGroup.Go(func() error { - return e.buildWithCallback(ctx, func(buildCtx context.Context, builtProject Project) error { + return e.buildWithCallback(ctx, func(buildCtx context.Context, module Module) error { deployGroup.Go(func() error { - e.projectsToBuild.Store(builtProject.Config().Key, false) - module, ok := builtProject.(Module) - if !ok { - // Skip deploying external libraries - return nil - } + e.modulesToBuild.Store(module.Config.Module, false) return Deploy(buildCtx, module, replicas, waitForDeployOnline, e.client) }) return nil - }, projects...) + }, moduleNames...) }) // Wait for all build and deploy attempts to complete @@ -431,9 +419,9 @@ func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDepl deployErr := deployGroup.Wait() pendingInitialBuilds := []string{} - e.projectsToBuild.Range(func(key ProjectKey, value bool) bool { + e.modulesToBuild.Range(func(name string, value bool) bool { if value { - pendingInitialBuilds = append(pendingInitialBuilds, string(key)) + pendingInitialBuilds = append(pendingInitialBuilds, name) } return true }) @@ -450,31 +438,31 @@ func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDepl return deployErr } -type buildCallback func(ctx context.Context, project Project) error +type buildCallback func(ctx context.Context, module Module) error -func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, projects ...ProjectKey) error { - mustBuild := map[ProjectKey]bool{} - if len(projects) == 0 { - e.projectMetas.Range(func(key ProjectKey, meta projectMeta) bool { - projects = append(projects, key) +func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, moduleNames ...string) error { + mustBuild := map[string]bool{} + if len(moduleNames) == 0 { + e.moduleMetas.Range(func(name string, meta moduleMeta) bool { + moduleNames = append(moduleNames, name) return true }) } - for _, key := range projects { - meta, ok := e.projectMetas.Load(key) + for _, name := range moduleNames { + meta, ok := e.moduleMetas.Load(name) if !ok { - return fmt.Errorf("project %q not found", key) + return fmt.Errorf("module %q not found", name) } // Update dependencies before building. var err error - project, err := UpdateDependencies(ctx, meta.project) + module, err := UpdateDependencies(ctx, meta.module) if err != nil { return err } - e.projectMetas.Store(key, projectMeta{project: project}) - mustBuild[key] = true + e.moduleMetas.Store(name, moduleMeta{module: module}) + mustBuild[name] = true } - graph, err := e.Graph(projects...) + graph, err := e.Graph(moduleNames...) if err != nil { return err } @@ -493,13 +481,11 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, wg := errgroup.Group{} wg.SetLimit(e.parallelism) - for _, keyStr := range group { - key := ProjectKey(keyStr) - + for _, moduleName := range group { wg.Go(func() error { - logger := log.FromContext(ctx).Scope(string(key)) + logger := log.FromContext(ctx).Scope(moduleName) ctx := log.ContextWithLogger(ctx, logger) - err := e.tryBuild(ctx, mustBuild, key, builtModules, schemas, callback) + err := e.tryBuild(ctx, mustBuild, moduleName, builtModules, schemas, callback) if err != nil { errCh <- err } @@ -532,19 +518,19 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, return nil } -func (e *Engine) tryBuild(ctx context.Context, mustBuild map[ProjectKey]bool, key ProjectKey, builtModules map[string]*schema.Module, schemas chan *schema.Module, callback buildCallback) error { +func (e *Engine) tryBuild(ctx context.Context, mustBuild map[string]bool, moduleName string, builtModules map[string]*schema.Module, schemas chan *schema.Module, callback buildCallback) error { logger := log.FromContext(ctx) - if !mustBuild[key] { - return e.mustSchema(ctx, key, builtModules, schemas) + if !mustBuild[moduleName] { + return e.mustSchema(ctx, moduleName, builtModules, schemas) } - meta, ok := e.projectMetas.Load(key) + meta, ok := e.moduleMetas.Load(moduleName) if !ok { - return fmt.Errorf("project %q not found", key) + return fmt.Errorf("Module %q not found", moduleName) } - for _, dep := range meta.project.Config().Dependencies { + for _, dep := range meta.module.Dependencies { if _, ok := builtModules[dep]; !ok { logger.Warnf("build skipped because dependency %q failed to build", dep) return nil @@ -552,74 +538,73 @@ func (e *Engine) tryBuild(ctx context.Context, mustBuild map[ProjectKey]bool, ke } meta.lastBuildStartTime = time.Now() - e.projectMetas.Store(key, meta) - err := e.build(ctx, key, builtModules, schemas) + e.moduleMetas.Store(moduleName, meta) + err := e.build(ctx, moduleName, builtModules, schemas) if err == nil && callback != nil { - return callback(ctx, meta.project) + return callback(ctx, meta.module) } return err } // Publish either the schema from the FTL controller, or from a local build. -func (e *Engine) mustSchema(ctx context.Context, key ProjectKey, builtModules map[string]*schema.Module, schemas chan<- *schema.Module) error { - if sch, ok := e.controllerSchema.Load(string(key)); ok { +func (e *Engine) mustSchema(ctx context.Context, moduleName string, builtModules map[string]*schema.Module, schemas chan<- *schema.Module) error { + if sch, ok := e.controllerSchema.Load(moduleName); ok { schemas <- sch return nil } - return e.build(ctx, key, builtModules, schemas) + return e.build(ctx, moduleName, builtModules, schemas) } // Build a module and publish its schema. // // Assumes that all dependencies have been built and are available in "built". -func (e *Engine) build(ctx context.Context, key ProjectKey, builtModules map[string]*schema.Module, schemas chan<- *schema.Module) error { - meta, ok := e.projectMetas.Load(key) +func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[string]*schema.Module, schemas chan<- *schema.Module) error { + meta, ok := e.moduleMetas.Load(moduleName) if !ok { - return fmt.Errorf("project %q not found", key) + return fmt.Errorf("module %q not found", moduleName) } combined := map[string]*schema.Module{} - if err := e.gatherSchemas(builtModules, meta.project, combined); err != nil { + if err := e.gatherSchemas(builtModules, meta.module, combined); err != nil { return err } sch := &schema.Schema{Modules: maps.Values(combined)} if e.listener != nil { - e.listener.OnBuildStarted(meta.project) + e.listener.OnBuildStarted(meta.module) } - err := Build(ctx, sch, meta.project, e.watcher.GetTransaction(meta.project.Config().Dir)) + err := Build(ctx, sch, meta.module, e.watcher.GetTransaction(meta.module.Config.Dir)) if err != nil { return err } - if module, ok := meta.project.(Module); ok { - moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(module.Dir, module.DeployDir, module.Schema)) - if err != nil { - return fmt.Errorf("could not load schema for module %q: %w", module.Config().Key, err) - } - schemas <- moduleSchema + config := meta.module.Config + moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(config.Dir, config.DeployDir, config.Schema)) + if err != nil { + return fmt.Errorf("could not load schema for module %q: %w", config.Module, err) } + schemas <- moduleSchema return nil } -// Construct a combined schema for a project and its transitive dependencies. +// Construct a combined schema for a module and its transitive dependencies. func (e *Engine) gatherSchemas( moduleSchemas map[string]*schema.Module, - project Project, + module Module, out map[string]*schema.Module, ) error { - latestModule, ok := e.projectMetas.Load(project.Config().Key) + latestModule, ok := e.moduleMetas.Load(module.Config.Module) if !ok { - latestModule = projectMeta{project: project} + latestModule = moduleMeta{module: module} } - for _, dep := range latestModule.project.Config().Dependencies { + for _, dep := range latestModule.module.Dependencies { out[dep] = moduleSchemas[dep] if dep != "builtin" { - depModule, ok := e.projectMetas.Load(ProjectKey(dep)) - // TODO: should we be gathering schemas from dependencies without a project? + depModule, ok := e.moduleMetas.Load(dep) + // TODO: should we be gathering schemas from dependencies without a module? // This can happen if the schema is loaded from the controller if ok { - if err := e.gatherSchemas(moduleSchemas, depModule.project, out); err != nil { + if err := e.gatherSchemas(moduleSchemas, depModule.module, out); err != nil { return err } } diff --git a/buildengine/engine_test.go b/buildengine/engine_test.go index cb7712ed8b..73a5211349 100644 --- a/buildengine/engine_test.go +++ b/buildengine/engine_test.go @@ -16,7 +16,7 @@ func TestEngine(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/projects/alpha", "testdata/projects/other", "testdata/projects/another"}, nil) + engine, err := buildengine.New(ctx, nil, []string{"testdata/alpha", "testdata/other", "testdata/another"}) assert.NoError(t, err) defer engine.Close() @@ -64,7 +64,7 @@ func TestCycleDetection(t *testing.T) { t.SkipNow() } ctx := log.ContextWithNewDefaultLogger(context.Background()) - engine, err := buildengine.New(ctx, nil, []string{"testdata/projects/depcycle1", "testdata/projects/depcycle2"}, nil) + engine, err := buildengine.New(ctx, nil, []string{"testdata/depcycle1", "testdata/depcycle2"}) assert.NoError(t, err) defer engine.Close() diff --git a/buildengine/filehash.go b/buildengine/filehash.go index d046b3102e..cfa12c2406 100644 --- a/buildengine/filehash.go +++ b/buildengine/filehash.go @@ -63,8 +63,8 @@ func CompareFileHashes(oldFiles, newFiles FileHashes) (FileChangeType, string, b // ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in // the given directory. -func ComputeFileHashes(project Project) (FileHashes, error) { - config := project.Config() +func ComputeFileHashes(module Module) (FileHashes, error) { + config := module.Config fileHashes := make(FileHashes) rootDirs := computeRootDirs(config.Dir, config.Watch) diff --git a/buildengine/module.go b/buildengine/module.go new file mode 100644 index 0000000000..45e0e9ecd7 --- /dev/null +++ b/buildengine/module.go @@ -0,0 +1,31 @@ +package buildengine + +import ( + "github.com/TBD54566975/ftl/common/moduleconfig" + "github.com/TBD54566975/ftl/internal/reflect" +) + +// Module represents an FTL module in the build engine +type Module struct { + Config moduleconfig.ModuleConfig + Dependencies []string +} + +func (m Module) CopyWithDependencies(dependencies []string) Module { + module := reflect.DeepCopy(m) + module.Dependencies = dependencies + return module +} + +// LoadModule loads a module from the given directory. +func LoadModule(dir string) (Module, error) { + config, err := moduleconfig.LoadModuleConfig(dir) + if err != nil { + return Module{}, err + } + module := Module{ + Config: config, + Dependencies: []string{}, + } + return module, nil +} diff --git a/buildengine/project.go b/buildengine/project.go deleted file mode 100644 index 065bf92bdb..0000000000 --- a/buildengine/project.go +++ /dev/null @@ -1,156 +0,0 @@ -package buildengine - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/TBD54566975/ftl/common/moduleconfig" - "github.com/TBD54566975/ftl/internal/reflect" -) - -// Project models FTL modules and external libraries and is used to manage dependencies within the build engine -// -//sumtype:decl -type Project interface { - sealed() - - Config() ProjectConfig - CopyWithDependencies(deps []string) Project - TypeString() string -} - -type ProjectConfig struct { - Key ProjectKey - Dir string - Language string - Watch []string - Dependencies []string -} - -var _ = (Project)(Module{}) -var _ = (Project)(ExternalLibrary{}) - -// Module represents an FTL module in the build engine -type Module struct { - moduleconfig.ModuleConfig - Dependencies []string -} - -func (m Module) sealed() {} - -func (m Module) Config() ProjectConfig { - return ProjectConfig{ - Key: ProjectKey(m.ModuleConfig.Module), - Dir: m.ModuleConfig.Dir, - Language: m.ModuleConfig.Language, - Watch: m.ModuleConfig.Watch, - Dependencies: m.Dependencies, - } -} - -func (m Module) CopyWithDependencies(dependencies []string) Project { - module := reflect.DeepCopy(m) - module.Dependencies = dependencies - return Project(module) -} - -func (m Module) TypeString() string { - return "module" -} - -// ExternalLibrary represents a library that makes use of FTL modules, but is not itself an FTL module -type ExternalLibrary struct { - Dir string - Language string - Dependencies []string -} - -func (e ExternalLibrary) sealed() {} - -func (e ExternalLibrary) Config() ProjectConfig { - var watch []string - switch e.Language { - case "go": - watch = []string{"**/*.go", "go.mod", "go.sum"} - case "kotlin": - watch = []string{"pom.xml", "src/**", "target/generated-sources"} - default: - panic(fmt.Sprintf("unknown language %T", e.Language)) - } - - return ProjectConfig{ - Key: ProjectKey("lib:" + e.Dir), - Dir: e.Dir, - Language: e.Language, - Watch: watch, - Dependencies: e.Dependencies, - } -} - -func (e ExternalLibrary) CopyWithDependencies(dependencies []string) Project { - lib := reflect.DeepCopy(e) - lib.Dependencies = dependencies - return Project(lib) -} - -func (e ExternalLibrary) TypeString() string { - return "library" -} - -// ProjectKey is a unique identifier for the project (ie: a module name or a library path) -// It is used to: -// - build the dependency graph -// - map changes in the file system to the project -type ProjectKey string - -func StringsFromProjectKeys(keys []ProjectKey) []string { - strs := make([]string, len(keys)) - for i, key := range keys { - strs[i] = string(key) - } - return strs -} - -func ProjectKeysFromModuleNames(names []string) []ProjectKey { - keys := make([]ProjectKey, len(names)) - for i, str := range names { - keys[i] = ProjectKey(str) - } - return keys -} - -// LoadModule loads a module from the given directory. -func LoadModule(dir string) (Module, error) { - config, err := moduleconfig.LoadModuleConfig(dir) - if err != nil { - return Module{}, err - } - module := Module{ModuleConfig: config} - return module, nil -} - -func LoadExternalLibrary(dir string) (ExternalLibrary, error) { - lib := ExternalLibrary{ - Dir: dir, - } - - goModPath := filepath.Join(dir, "go.mod") - pomPath := filepath.Join(dir, "pom.xml") - if _, err := os.Stat(goModPath); err == nil { - lib.Language = "go" - } else if !os.IsNotExist(err) { - return ExternalLibrary{}, err - } else { - if _, err = os.Stat(pomPath); err == nil { - lib.Language = "kotlin" - } else if !os.IsNotExist(err) { - return ExternalLibrary{}, err - } - } - if lib.Language == "" { - return ExternalLibrary{}, fmt.Errorf("could not autodetect language: no go.mod or pom.xml found in %s", dir) - } - - return lib, nil -} diff --git a/buildengine/testdata/projects/alpha/alpha.go b/buildengine/testdata/alpha/alpha.go similarity index 100% rename from buildengine/testdata/projects/alpha/alpha.go rename to buildengine/testdata/alpha/alpha.go diff --git a/buildengine/testdata/projects/alpha/ftl.toml b/buildengine/testdata/alpha/ftl.toml similarity index 100% rename from buildengine/testdata/projects/alpha/ftl.toml rename to buildengine/testdata/alpha/ftl.toml diff --git a/buildengine/testdata/projects/alpha/go.mod b/buildengine/testdata/alpha/go.mod similarity index 97% rename from buildengine/testdata/projects/alpha/go.mod rename to buildengine/testdata/alpha/go.mod index 8729433953..638def5907 100644 --- a/buildengine/testdata/projects/alpha/go.mod +++ b/buildengine/testdata/alpha/go.mod @@ -41,4 +41,4 @@ require ( google.golang.org/protobuf v1.34.2 // indirect ) -replace github.com/TBD54566975/ftl => ../../../.. +replace github.com/TBD54566975/ftl => ../../.. diff --git a/buildengine/testdata/projects/alpha/go.sum b/buildengine/testdata/alpha/go.sum similarity index 100% rename from buildengine/testdata/projects/alpha/go.sum rename to buildengine/testdata/alpha/go.sum diff --git a/buildengine/testdata/projects/alpha/pkg/pkg.go b/buildengine/testdata/alpha/pkg/pkg.go similarity index 100% rename from buildengine/testdata/projects/alpha/pkg/pkg.go rename to buildengine/testdata/alpha/pkg/pkg.go diff --git a/buildengine/testdata/projects/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt b/buildengine/testdata/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt similarity index 100% rename from buildengine/testdata/projects/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt rename to buildengine/testdata/alphakotlin/src/main/kotlin/ftl/alpha/Alpha.kt diff --git a/buildengine/testdata/projects/another/another.go b/buildengine/testdata/another/another.go similarity index 100% rename from buildengine/testdata/projects/another/another.go rename to buildengine/testdata/another/another.go diff --git a/buildengine/testdata/projects/another/ftl.toml b/buildengine/testdata/another/ftl.toml similarity index 100% rename from buildengine/testdata/projects/another/ftl.toml rename to buildengine/testdata/another/ftl.toml diff --git a/buildengine/testdata/projects/another/go.mod b/buildengine/testdata/another/go.mod similarity index 97% rename from buildengine/testdata/projects/another/go.mod rename to buildengine/testdata/another/go.mod index cff0f9de1e..cb210b13a2 100644 --- a/buildengine/testdata/projects/another/go.mod +++ b/buildengine/testdata/another/go.mod @@ -41,4 +41,4 @@ require ( google.golang.org/protobuf v1.34.2 // indirect ) -replace github.com/TBD54566975/ftl => ../../../.. +replace github.com/TBD54566975/ftl => ../../.. diff --git a/buildengine/testdata/projects/another/go.sum b/buildengine/testdata/another/go.sum similarity index 100% rename from buildengine/testdata/projects/another/go.sum rename to buildengine/testdata/another/go.sum diff --git a/buildengine/testdata/projects/depcycle1/depcycle1.go b/buildengine/testdata/depcycle1/depcycle1.go similarity index 100% rename from buildengine/testdata/projects/depcycle1/depcycle1.go rename to buildengine/testdata/depcycle1/depcycle1.go diff --git a/buildengine/testdata/projects/depcycle1/ftl.toml b/buildengine/testdata/depcycle1/ftl.toml similarity index 100% rename from buildengine/testdata/projects/depcycle1/ftl.toml rename to buildengine/testdata/depcycle1/ftl.toml diff --git a/buildengine/testdata/projects/depcycle1/go.mod b/buildengine/testdata/depcycle1/go.mod similarity index 100% rename from buildengine/testdata/projects/depcycle1/go.mod rename to buildengine/testdata/depcycle1/go.mod diff --git a/buildengine/testdata/projects/depcycle2/depcycle2.go b/buildengine/testdata/depcycle2/depcycle2.go similarity index 100% rename from buildengine/testdata/projects/depcycle2/depcycle2.go rename to buildengine/testdata/depcycle2/depcycle2.go diff --git a/buildengine/testdata/projects/depcycle2/ftl.toml b/buildengine/testdata/depcycle2/ftl.toml similarity index 100% rename from buildengine/testdata/projects/depcycle2/ftl.toml rename to buildengine/testdata/depcycle2/ftl.toml diff --git a/buildengine/testdata/projects/depcycle2/go.mod b/buildengine/testdata/depcycle2/go.mod similarity index 100% rename from buildengine/testdata/projects/depcycle2/go.mod rename to buildengine/testdata/depcycle2/go.mod diff --git a/buildengine/testdata/projects/echokotlin/ftl.toml b/buildengine/testdata/echokotlin/ftl.toml similarity index 100% rename from buildengine/testdata/projects/echokotlin/ftl.toml rename to buildengine/testdata/echokotlin/ftl.toml diff --git a/buildengine/testdata/projects/echokotlin/pom.xml b/buildengine/testdata/echokotlin/pom.xml similarity index 100% rename from buildengine/testdata/projects/echokotlin/pom.xml rename to buildengine/testdata/echokotlin/pom.xml diff --git a/buildengine/testdata/projects/echokotlin/src/main/kotlin/ftl/echo/Echo.kt b/buildengine/testdata/echokotlin/src/main/kotlin/ftl/echo/Echo.kt similarity index 100% rename from buildengine/testdata/projects/echokotlin/src/main/kotlin/ftl/echo/Echo.kt rename to buildengine/testdata/echokotlin/src/main/kotlin/ftl/echo/Echo.kt diff --git a/buildengine/testdata/projects/external/external.go b/buildengine/testdata/external/external.go similarity index 100% rename from buildengine/testdata/projects/external/external.go rename to buildengine/testdata/external/external.go diff --git a/buildengine/testdata/projects/external/ftl.toml b/buildengine/testdata/external/ftl.toml similarity index 100% rename from buildengine/testdata/projects/external/ftl.toml rename to buildengine/testdata/external/ftl.toml diff --git a/buildengine/testdata/projects/lib/go.mod b/buildengine/testdata/external/go.mod similarity index 74% rename from buildengine/testdata/projects/lib/go.mod rename to buildengine/testdata/external/go.mod index bfbc970bf3..33c237f508 100644 --- a/buildengine/testdata/projects/lib/go.mod +++ b/buildengine/testdata/external/go.mod @@ -1,4 +1,4 @@ -module lib +module ftl/external go 1.22.2 diff --git a/buildengine/testdata/projects/external/go.sum b/buildengine/testdata/external/go.sum similarity index 100% rename from buildengine/testdata/projects/external/go.sum rename to buildengine/testdata/external/go.sum diff --git a/buildengine/testdata/projects/externalkotlin/ftl.toml b/buildengine/testdata/externalkotlin/ftl.toml similarity index 100% rename from buildengine/testdata/projects/externalkotlin/ftl.toml rename to buildengine/testdata/externalkotlin/ftl.toml diff --git a/buildengine/testdata/projects/externalkotlin/pom.xml b/buildengine/testdata/externalkotlin/pom.xml similarity index 100% rename from buildengine/testdata/projects/externalkotlin/pom.xml rename to buildengine/testdata/externalkotlin/pom.xml diff --git a/buildengine/testdata/projects/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt b/buildengine/testdata/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt similarity index 100% rename from buildengine/testdata/projects/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt rename to buildengine/testdata/externalkotlin/src/main/kotlin/ftl/externalkotlin/ExternalKotlin.kt diff --git a/buildengine/testdata/projects/highgoversion/ftl.toml b/buildengine/testdata/highgoversion/ftl.toml similarity index 100% rename from buildengine/testdata/projects/highgoversion/ftl.toml rename to buildengine/testdata/highgoversion/ftl.toml diff --git a/buildengine/testdata/projects/highgoversion/go.mod b/buildengine/testdata/highgoversion/go.mod similarity index 97% rename from buildengine/testdata/projects/highgoversion/go.mod rename to buildengine/testdata/highgoversion/go.mod index f8f5d9d31b..a3bb856861 100644 --- a/buildengine/testdata/projects/highgoversion/go.mod +++ b/buildengine/testdata/highgoversion/go.mod @@ -43,4 +43,4 @@ require ( google.golang.org/protobuf v1.33.0 // indirect ) -replace github.com/TBD54566975/ftl => ../../../.. +replace github.com/TBD54566975/ftl => ../../.. diff --git a/buildengine/testdata/projects/highgoversion/go.sum b/buildengine/testdata/highgoversion/go.sum similarity index 100% rename from buildengine/testdata/projects/highgoversion/go.sum rename to buildengine/testdata/highgoversion/go.sum diff --git a/buildengine/testdata/projects/highgoversion/highgoversion.go b/buildengine/testdata/highgoversion/highgoversion.go similarity index 100% rename from buildengine/testdata/projects/highgoversion/highgoversion.go rename to buildengine/testdata/highgoversion/highgoversion.go diff --git a/buildengine/testdata/projects/other/ftl.toml b/buildengine/testdata/other/ftl.toml similarity index 100% rename from buildengine/testdata/projects/other/ftl.toml rename to buildengine/testdata/other/ftl.toml diff --git a/buildengine/testdata/projects/other/go.mod b/buildengine/testdata/other/go.mod similarity index 97% rename from buildengine/testdata/projects/other/go.mod rename to buildengine/testdata/other/go.mod index ca034890a5..496a8b03b9 100644 --- a/buildengine/testdata/projects/other/go.mod +++ b/buildengine/testdata/other/go.mod @@ -41,4 +41,4 @@ require ( google.golang.org/protobuf v1.34.2 // indirect ) -replace github.com/TBD54566975/ftl => ../../../.. +replace github.com/TBD54566975/ftl => ../../.. diff --git a/buildengine/testdata/projects/other/go.sum b/buildengine/testdata/other/go.sum similarity index 100% rename from buildengine/testdata/projects/other/go.sum rename to buildengine/testdata/other/go.sum diff --git a/buildengine/testdata/projects/other/other.go b/buildengine/testdata/other/other.go similarity index 100% rename from buildengine/testdata/projects/other/other.go rename to buildengine/testdata/other/other.go diff --git a/buildengine/testdata/projects/external/go.mod b/buildengine/testdata/projects/external/go.mod deleted file mode 100644 index bcc8e68820..0000000000 --- a/buildengine/testdata/projects/external/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module ftl/external - -go 1.22.2 - -replace github.com/TBD54566975/ftl => ../../../.. diff --git a/buildengine/testdata/projects/lib/lib.go b/buildengine/testdata/projects/lib/lib.go deleted file mode 100644 index 66e97fc63e..0000000000 --- a/buildengine/testdata/projects/lib/lib.go +++ /dev/null @@ -1,11 +0,0 @@ -package lib - -import ( - "context" - "fmt" - "ftl/alpha" -) - -func CreateEchoResponse(ctx context.Context, req alpha.EchoRequest) alpha.EchoResponse { - return alpha.EchoResponse{Message: fmt.Sprintf("Hello, %s!!!", req.Name)} -} diff --git a/buildengine/testdata/projects/libkotlin/pom.xml b/buildengine/testdata/projects/libkotlin/pom.xml deleted file mode 100644 index 34b9a2ecca..0000000000 --- a/buildengine/testdata/projects/libkotlin/pom.xml +++ /dev/null @@ -1,153 +0,0 @@ - - - 4.0.0 - - ftl - lib - 1.0-SNAPSHOT - - - 1.0-SNAPSHOT - 1.8 - 2.0.0 - true - ${java.version} - ${java.version} - - - - - org.jetbrains.kotlin - kotlin-stdlib - ${kotlin.version} - - - xyz.block - ftl-runtime - ${ftl.version} - - - org.postgresql - postgresql - 42.7.3 - - - - - - - - kotlin-maven-plugin - org.jetbrains.kotlin - ${kotlin.version} - - - compile - - compile - - - - ${project.basedir}/src/main/kotlin - - - - - test-compile - - test-compile - - - - ${project.basedir}/src/test/kotlin - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.7.0 - - - - copy-dependencies - compile - - copy-dependencies - - - ${project.build.directory}/dependency - runtime - - - - - build-classpath - compile - - build-classpath - - - ${project.build.directory}/classpath.txt - dependency - - - - build-classpath-property - compile - - build-classpath - - - generated.classpath - ${project.build.directory}/dependency - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.6.0 - - - generate-sources - - add-source - - - - ${project.build.directory}/generated-sources/ftl - - - - - - - - - - - kotlin-maven-plugin - org.jetbrains.kotlin - - - org.apache.maven.plugins - maven-dependency-plugin - - - - org.codehaus.mojo - build-helper-maven-plugin - - - - com.github.ozsie - detekt-maven-plugin - - - - \ No newline at end of file diff --git a/buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt b/buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt deleted file mode 100644 index 5bead81853..0000000000 --- a/buildengine/testdata/projects/libkotlin/src/main/kotlin/lib/Lib.kt +++ /dev/null @@ -1,11 +0,0 @@ -package lib - -import ftl.builtin.Empty -import ftl.echo.EchoRequest -import xyz.block.ftl.Context -import xyz.block.ftl.Verb - - -fun generateEchoText(req: EchoRequest): String { - return "Hello" -} diff --git a/buildengine/watch.go b/buildengine/watch.go index e8fa25ac93..f599ccd6d9 100644 --- a/buildengine/watch.go +++ b/buildengine/watch.go @@ -13,44 +13,44 @@ import ( "github.com/TBD54566975/ftl/internal/maps" ) -// A WatchEvent is an event that occurs when a project is added, removed, or +// A WatchEvent is an event that occurs when a module is added, removed, or // changed. type WatchEvent interface{ watchEvent() } -type WatchEventProjectAdded struct{ Project Project } +type WatchEventModuleAdded struct{ Module Module } -func (WatchEventProjectAdded) watchEvent() {} +func (WatchEventModuleAdded) watchEvent() {} -type WatchEventProjectRemoved struct{ Project Project } +type WatchEventModuleRemoved struct{ Module Module } -func (WatchEventProjectRemoved) watchEvent() {} +func (WatchEventModuleRemoved) watchEvent() {} -type WatchEventProjectChanged struct { - Project Project - Change FileChangeType - Path string - Time time.Time +type WatchEventModuleChanged struct { + Module Module + Change FileChangeType + Path string + Time time.Time } -func (WatchEventProjectChanged) watchEvent() {} +func (WatchEventModuleChanged) watchEvent() {} -type projectHashes struct { - Hashes FileHashes - Project Project +type moduleHashes struct { + Hashes FileHashes + Module Module } type Watcher struct { isWatching bool - // use mutex whenever accessing / modifying existingProjects or moduleTransactions + // use mutex whenever accessing / modifying existingModules or moduleTransactions mutex sync.Mutex - existingProjects map[string]projectHashes + existingModules map[string]moduleHashes moduleTransactions map[string][]*modifyFilesTransaction } func NewWatcher() *Watcher { svc := &Watcher{ - existingProjects: map[string]projectHashes{}, + existingModules: map[string]moduleHashes{}, moduleTransactions: map[string][]*modifyFilesTransaction{}, } @@ -64,9 +64,9 @@ func (w *Watcher) GetTransaction(moduleDir string) ModifyFilesTransaction { } } -// Watch the given directories for new projects, deleted projects, and changes to -// existing projects, publishing a change event for each. -func (w *Watcher) Watch(ctx context.Context, period time.Duration, moduleDirs []string, externalLibDirs []string) (*pubsub.Topic[WatchEvent], error) { +// Watch the given directories for new modules, deleted modules, and changes to +// existing modules, publishing a change event for each. +func (w *Watcher) Watch(ctx context.Context, period time.Duration, moduleDirs []string) (*pubsub.Topic[WatchEvent], error) { if w.isWatching { return nil, fmt.Errorf("file watcher is already watching") } @@ -89,58 +89,58 @@ func (w *Watcher) Watch(ctx context.Context, period time.Duration, moduleDirs [] return } - projects, err := DiscoverProjects(ctx, moduleDirs, externalLibDirs) + modules, err := DiscoverModules(ctx, moduleDirs) if err != nil { - logger.Tracef("error discovering projects: %v", err) + logger.Tracef("error discovering modules: %v", err) continue } - projectsByDir := maps.FromSlice(projects, func(project Project) (string, Project) { - return project.Config().Dir, project + modulesByDir := maps.FromSlice(modules, func(module Module) (string, Module) { + return module.Config.Dir, module }) w.mutex.Lock() - // Trigger events for removed projects. - for _, existingProject := range w.existingProjects { - if transactions, ok := w.moduleTransactions[existingProject.Project.Config().Dir]; ok && len(transactions) > 0 { - // Skip projects that currently have transactions + // Trigger events for removed modules. + for _, existingModule := range w.existingModules { + if transactions, ok := w.moduleTransactions[existingModule.Module.Config.Dir]; ok && len(transactions) > 0 { + // Skip modules that currently have transactions continue } - existingConfig := existingProject.Project.Config() - if _, haveProject := projectsByDir[existingConfig.Dir]; !haveProject { - logger.Debugf("removed %s %q", existingProject.Project.TypeString(), existingProject.Project.Config().Key) - topic.Publish(WatchEventProjectRemoved{Project: existingProject.Project}) - delete(w.existingProjects, existingConfig.Dir) + existingConfig := existingModule.Module.Config + if _, haveModule := modulesByDir[existingConfig.Dir]; !haveModule { + logger.Debugf("removed %q", existingModule.Module.Config.Module) + topic.Publish(WatchEventModuleRemoved{Module: existingModule.Module}) + delete(w.existingModules, existingConfig.Dir) } } - // Compare the projects to the existing projects. - for _, project := range projectsByDir { - if transactions, ok := w.moduleTransactions[project.Config().Dir]; ok && len(transactions) > 0 { - // Skip projects that currently have transactions + // Compare the modules to the existing modules. + for _, module := range modulesByDir { + config := module.Config + if transactions, ok := w.moduleTransactions[config.Dir]; ok && len(transactions) > 0 { + // Skip modules that currently have transactions continue } - config := project.Config() - existingProject, haveExistingProject := w.existingProjects[config.Dir] - hashes, err := ComputeFileHashes(project) + existingModule, haveExistingModule := w.existingModules[config.Dir] + hashes, err := ComputeFileHashes(module) if err != nil { logger.Tracef("error computing file hashes for %s: %v", config.Dir, err) continue } - if haveExistingProject { - changeType, path, equal := CompareFileHashes(existingProject.Hashes, hashes) + if haveExistingModule { + changeType, path, equal := CompareFileHashes(existingModule.Hashes, hashes) if equal { continue } - logger.Debugf("changed %s %q: %c%s", project.TypeString(), project.Config().Key, changeType, path) - topic.Publish(WatchEventProjectChanged{Project: existingProject.Project, Change: changeType, Path: path, Time: time.Now()}) - w.existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: existingProject.Project} + logger.Debugf("changed %q: %c%s", config.Module, changeType, path) + topic.Publish(WatchEventModuleChanged{Module: existingModule.Module, Change: changeType, Path: path, Time: time.Now()}) + w.existingModules[config.Dir] = moduleHashes{Hashes: hashes, Module: existingModule.Module} continue } - logger.Debugf("added %s %q", project.TypeString(), project.Config().Key) - topic.Publish(WatchEventProjectAdded{Project: project}) - w.existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: project} + logger.Debugf("added %q", config.Module) + topic.Publish(WatchEventModuleAdded{Module: module}) + w.existingModules[config.Dir] = moduleHashes{Hashes: hashes, Module: module} } w.mutex.Unlock() } @@ -207,14 +207,14 @@ func (t *modifyFilesTransaction) ModifiedFiles(paths ...string) error { t.watcher.mutex.Lock() defer t.watcher.mutex.Unlock() - projectHashes, ok := t.watcher.existingProjects[t.moduleDir] + moduleHashes, ok := t.watcher.existingModules[t.moduleDir] if !ok { - // skip updating hashes because we have not discovered this project yet + // skip updating hashes because we have not discovered this module yet return nil } for _, path := range paths { - hash, matched, err := ComputeFileHash(projectHashes.Project.Config().Dir, path, projectHashes.Project.Config().Watch) + hash, matched, err := ComputeFileHash(moduleHashes.Module.Config.Dir, path, moduleHashes.Module.Config.Watch) if err != nil { return err } @@ -222,9 +222,9 @@ func (t *modifyFilesTransaction) ModifiedFiles(paths ...string) error { continue } - projectHashes.Hashes[path] = hash + moduleHashes.Hashes[path] = hash } - t.watcher.existingProjects[t.moduleDir] = projectHashes + t.watcher.existingModules[t.moduleDir] = moduleHashes return nil } diff --git a/buildengine/watch_test.go b/buildengine/watch_test.go index 5d32d2e845..5a5d00ce61 100644 --- a/buildengine/watch_test.go +++ b/buildengine/watch_test.go @@ -43,8 +43,8 @@ func TestWatch(t *testing.T) { two := loadModule(t, dir, "two") waitForEvents(t, events, []WatchEvent{ - WatchEventProjectAdded{Project: one}, - WatchEventProjectAdded{Project: two}, + WatchEventModuleAdded{Module: one}, + WatchEventModuleAdded{Module: two}, }) // Delete a module @@ -55,8 +55,8 @@ func TestWatch(t *testing.T) { updateModFile(t, filepath.Join(dir, "one")) waitForEvents(t, events, []WatchEvent{ - WatchEventProjectChanged{Project: one}, - WatchEventProjectRemoved{Project: two}, + WatchEventModuleChanged{Module: one}, + WatchEventModuleRemoved{Module: two}, }) topic.Close() } @@ -80,7 +80,7 @@ func TestWatchWithBuildModifyingFiles(t *testing.T) { events, topic := startWatching(ctx, t, w, dir) waitForEvents(t, events, []WatchEvent{ - WatchEventProjectAdded{Project: loadModule(t, dir, "one")}, + WatchEventModuleAdded{Module: loadModule(t, dir, "one")}, }) // Change a file in a module, within a transaction @@ -118,7 +118,7 @@ func TestWatchWithBuildAndUserModifyingFiles(t *testing.T) { events, topic := startWatching(ctx, t, w, dir) waitForEvents(t, events, []WatchEvent{ - WatchEventProjectAdded{Project: one}, + WatchEventModuleAdded{Module: one}, }) // Change a file in a module, within a transaction @@ -140,7 +140,7 @@ func TestWatchWithBuildAndUserModifyingFiles(t *testing.T) { assert.NoError(t, err) waitForEvents(t, events, []WatchEvent{ - WatchEventProjectChanged{Project: one}, + WatchEventModuleChanged{Module: one}, }) topic.Close() } @@ -150,14 +150,14 @@ func loadModule(t *testing.T, dir, name string) Module { config, err := moduleconfig.LoadModuleConfig(filepath.Join(dir, name)) assert.NoError(t, err) return Module{ - ModuleConfig: config, + Config: config, } } func startWatching(ctx context.Context, t *testing.T, w *Watcher, dir string) (chan WatchEvent, *pubsub.Topic[WatchEvent]) { t.Helper() events := make(chan WatchEvent, 128) - topic, err := w.Watch(ctx, pollFrequency, []string{dir}, nil) + topic, err := w.Watch(ctx, pollFrequency, []string{dir}) assert.NoError(t, err) topic.Subscribe(events) @@ -199,12 +199,12 @@ func waitForEvents(t *testing.T, events chan WatchEvent, expected []WatchEvent) func keyForEvent(event WatchEvent) string { switch event := event.(type) { - case WatchEventProjectAdded: - return "added:" + string(event.Project.Config().Key) - case WatchEventProjectRemoved: - return "removed:" + string(event.Project.Config().Key) - case WatchEventProjectChanged: - return "updated:" + string(event.Project.Config().Key) + case WatchEventModuleAdded: + return "added:" + event.Module.Config.Module + case WatchEventModuleRemoved: + return "removed:" + event.Module.Config.Module + case WatchEventModuleChanged: + return "updated:" + event.Module.Config.Module default: panic("unknown event type") } diff --git a/cmd/ftl/cmd_build.go b/cmd/ftl/cmd_build.go index f5622c8b59..20be5fddee 100644 --- a/cmd/ftl/cmd_build.go +++ b/cmd/ftl/cmd_build.go @@ -13,19 +13,17 @@ import ( type buildCmd struct { Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` Dirs []string `arg:"" help:"Base directories containing modules." type:"existingdir" optional:""` - External []string `help:"Directories for libraries that require FTL module stubs." type:"existingdir" optional:""` } func (b *buildCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - if len(b.Dirs) == 0 && len(b.External) == 0 { + if len(b.Dirs) == 0 { b.Dirs = projConfig.AbsModuleDirs() - b.External = projConfig.ExternalDirs } - if len(b.Dirs) == 0 && len(b.External) == 0 { + if len(b.Dirs) == 0 { return errors.New("no directories specified") } - engine, err := buildengine.New(ctx, client, b.Dirs, b.External, buildengine.Parallelism(b.Parallelism)) + engine, err := buildengine.New(ctx, client, b.Dirs, buildengine.Parallelism(b.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_deploy.go b/cmd/ftl/cmd_deploy.go index 65ff8bb86d..8ea47241c3 100644 --- a/cmd/ftl/cmd_deploy.go +++ b/cmd/ftl/cmd_deploy.go @@ -17,7 +17,7 @@ type deployCmd struct { func (d *deployCmd) Run(ctx context.Context) error { client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx) - engine, err := buildengine.New(ctx, client, d.Dirs, []string{}, buildengine.Parallelism(d.Parallelism)) + engine, err := buildengine.New(ctx, client, d.Dirs, buildengine.Parallelism(d.Parallelism)) if err != nil { return err } diff --git a/cmd/ftl/cmd_dev.go b/cmd/ftl/cmd_dev.go index e8ea1ed9ef..929a71c13b 100644 --- a/cmd/ftl/cmd_dev.go +++ b/cmd/ftl/cmd_dev.go @@ -19,7 +19,6 @@ import ( type devCmd struct { Parallelism int `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"` Dirs []string `arg:"" help:"Base directories containing modules." type:"existingdir" optional:""` - External []string `help:"Directories for libraries that require FTL module stubs." type:"existingdir" optional:""` Watch time.Duration `help:"Watch template directory at this frequency and regenerate on change." default:"500ms"` NoServe bool `help:"Do not start the FTL server." default:"false"` Lsp bool `help:"Run the language server." default:"false"` @@ -29,11 +28,10 @@ type devCmd struct { } func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error { - if len(d.Dirs) == 0 && len(d.External) == 0 { + if len(d.Dirs) == 0 { d.Dirs = projConfig.AbsModuleDirs() - d.External = projConfig.ExternalDirs } - if len(d.Dirs) == 0 && len(d.External) == 0 { + if len(d.Dirs) == 0 { return errors.New("no directories specified") } @@ -86,7 +84,7 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error }) } - engine, err := buildengine.New(ctx, client, d.Dirs, d.External, opts...) + engine, err := buildengine.New(ctx, client, d.Dirs, opts...) if err != nil { return err } @@ -96,6 +94,6 @@ func (d *devCmd) Run(ctx context.Context, projConfig projectconfig.Config) error return g.Wait() } -func (d *devCmd) OnBuildStarted(project buildengine.Project) { - d.languageServer.BuildStarted(project.Config().Dir) +func (d *devCmd) OnBuildStarted(module buildengine.Module) { + d.languageServer.BuildStarted(module.Config.Dir) } diff --git a/common/projectconfig/projectconfig.go b/common/projectconfig/projectconfig.go index 68ad7a66eb..c6fb4070bc 100644 --- a/common/projectconfig/projectconfig.go +++ b/common/projectconfig/projectconfig.go @@ -32,7 +32,6 @@ type Config struct { Global ConfigAndSecrets `toml:"global"` Modules map[string]ConfigAndSecrets `toml:"modules"` ModuleDirs []string `toml:"module-dirs"` - ExternalDirs []string `toml:"external-dirs"` Commands Commands `toml:"commands"` FTLMinVersion string `toml:"ftl-min-version"` } diff --git a/common/projectconfig/projectconfig_test.go b/common/projectconfig/projectconfig_test.go index f4a7150769..40e0d9a124 100644 --- a/common/projectconfig/projectconfig_test.go +++ b/common/projectconfig/projectconfig_test.go @@ -26,8 +26,7 @@ func TestProjectConfig(t *testing.T) { }, }, }, - ModuleDirs: []string{"a/b/c", "d"}, - ExternalDirs: []string{"e/f", "g/h"}, + ModuleDirs: []string{"a/b/c", "d"}, Commands: Commands{ Startup: []string{"echo 'Executing global pre-build command'"}, }, diff --git a/common/projectconfig/testdata/ftl-project.toml b/common/projectconfig/testdata/ftl-project.toml index bdbd58f86a..16bb59dbaf 100644 --- a/common/projectconfig/testdata/ftl-project.toml +++ b/common/projectconfig/testdata/ftl-project.toml @@ -1,5 +1,4 @@ module-dirs = ["a/b/c", "d"] -external-dirs = ["e/f", "g/h"] [modules.module.configuration] githubAccessToken = "keychain://githubAccessToken" diff --git a/common/projectconfig/testdata/withMinVersion/ftl-project.toml b/common/projectconfig/testdata/withMinVersion/ftl-project.toml index f5d6824086..fc1aa653af 100644 --- a/common/projectconfig/testdata/withMinVersion/ftl-project.toml +++ b/common/projectconfig/testdata/withMinVersion/ftl-project.toml @@ -1,5 +1,4 @@ module-dirs = ["a/b/c", "d"] -external-dirs = ["e/f", "g/h"] ftl-min-version = "0.129.2" [modules.module.configuration] diff --git a/ftl-project.toml b/ftl-project.toml index fd63b39f22..951e661e64 100644 --- a/ftl-project.toml +++ b/ftl-project.toml @@ -1,5 +1,4 @@ module-dirs = ["examples/go"] -external-dirs = [] ftl-min-version = "" [global]