diff --git a/buildengine/build.go b/buildengine/build.go index 02c7dc9e5e..df36fad42d 100644 --- a/buildengine/build.go +++ b/buildengine/build.go @@ -13,7 +13,7 @@ func Build(ctx context.Context, sch *schema.Schema, module Module) error { logger := log.FromContext(ctx).Scope(module.Module) ctx = log.ContextWithLogger(ctx, logger) logger.Infof("Building module") - switch module.Language() { + switch module.Language { case "go": return buildGo(ctx, sch, module) @@ -21,6 +21,6 @@ func Build(ctx context.Context, sch *schema.Schema, module Module) error { return buildKotlin(ctx, sch, module) default: - return fmt.Errorf("unknown language %q", module.Language()) + return fmt.Errorf("unknown language %q", module.Language) } } diff --git a/buildengine/build_go.go b/buildengine/build_go.go index 0f46c5009a..3e06839e16 100644 --- a/buildengine/build_go.go +++ b/buildengine/build_go.go @@ -9,8 +9,8 @@ import ( ) func buildGo(ctx context.Context, sch *schema.Schema, module Module) error { - if err := compile.Build(ctx, module.Dir(), sch); err != nil { - return fmt.Errorf("failed to build module %s: %w", module.Key(), err) + if err := compile.Build(ctx, module.Dir, sch); err != nil { + return fmt.Errorf("failed to build %q: %w", module, err) } return nil } diff --git a/buildengine/build_kotlin.go b/buildengine/build_kotlin.go index 771d50b47f..5f7853bd1e 100644 --- a/buildengine/build_kotlin.go +++ b/buildengine/build_kotlin.go @@ -44,7 +44,7 @@ func (e externalModuleContext) ExternalModules() []*schema.Module { func buildKotlin(ctx context.Context, sch *schema.Schema, module Module) error { logger := log.FromContext(ctx) - if err := SetPOMProperties(ctx, module.Dir()); err != nil { + if err := SetPOMProperties(ctx, module.Dir); err != nil { return fmt.Errorf("unable to update ftl.version in %s: %w", module.Dir, err) } @@ -57,7 +57,7 @@ func buildKotlin(ctx context.Context, sch *schema.Schema, module Module) error { } logger.Debugf("Using build command '%s'", module.Build) - err := exec.Command(ctx, log.Debug, module.Dir(), "bash", "-c", module.Build).RunBuffered(ctx) + err := exec.Command(ctx, log.Debug, module.Dir, "bash", "-c", module.Build).RunBuffered(ctx) if err != nil { return fmt.Errorf("failed to build module %s: %w", module.Module, err) } @@ -128,12 +128,13 @@ exec java -cp "classes:$(cat classpath.txt)" xyz.block.ftl.main.MainKt func generateExternalModules(ctx context.Context, project Project, sch *schema.Schema) error { logger := log.FromContext(ctx) funcs := maps.Clone(scaffoldFuncs) + config := project.Config() // Wipe the modules directory to ensure we don't have any stale modules. - _ = os.RemoveAll(filepath.Join(project.Dir(), "target", "generated-sources", "ftl")) + _ = os.RemoveAll(filepath.Join(config.Dir, "target", "generated-sources", "ftl")) logger.Debugf("Generating external modules") - return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), project.Dir(), externalModuleContext{ + return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), config.Dir, externalModuleContext{ project: project, Schema: sch, }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) diff --git a/buildengine/deps.go b/buildengine/deps.go index d200fb85ba..f1f5473f3b 100644 --- a/buildengine/deps.go +++ b/buildengine/deps.go @@ -23,7 +23,7 @@ import ( // Project with those dependencies populated. func UpdateDependencies(ctx context.Context, project Project) (Project, error) { logger := log.FromContext(ctx) - logger.Debugf("Extracting dependencies for module %s", project.Key()) + logger.Debugf("Extracting dependencies for %s", project) dependencies, err := extractDependencies(project) if err != nil { return Project(&Module{}), err @@ -43,24 +43,25 @@ func UpdateDependencies(ctx context.Context, project Project) (Project, error) { return out, nil } -func extractDependencies(project Project) ([]string, error) { +func extractDependencies(project Project) ([]ProjectKey, error) { + config := project.Config() name := "" if config, ok := project.(Module); ok { name = config.Module } - switch project.Language() { + switch config.Language { case "go": - return extractGoFTLImports(name, project.Dir()) + return extractGoFTLImports(name, config.Dir) case "kotlin": - return extractKotlinFTLImports(name, project.Dir()) + return extractKotlinFTLImports(name, config.Dir) default: - return nil, fmt.Errorf("unsupported language: %s", project.Language()) + return nil, fmt.Errorf("unsupported language: %s", config.Language) } } -func extractGoFTLImports(self, dir string) ([]string, error) { +func extractGoFTLImports(self, dir string) ([]ProjectKey, error) { dependencies := map[string]bool{} fset := token.NewFileSet() err := WalkDir(dir, func(path string, d fs.DirEntry) error { @@ -99,10 +100,10 @@ func extractGoFTLImports(self, dir string) ([]string, error) { } modules := maps.Keys(dependencies) sort.Strings(modules) - return modules, nil + return ProjectKeysFromModuleNames(modules), nil } -func extractKotlinFTLImports(self, dir string) ([]string, error) { +func extractKotlinFTLImports(self, dir string) ([]ProjectKey, error) { dependencies := map[string]bool{} kotlinImportRegex := regexp.MustCompile(`^import ftl\.([A-Za-z0-9_.]+)`) @@ -138,5 +139,5 @@ func extractKotlinFTLImports(self, dir string) ([]string, error) { } modules := maps.Keys(dependencies) sort.Strings(modules) - return modules, nil + return ProjectKeysFromModuleNames(modules), nil } diff --git a/buildengine/discover.go b/buildengine/discover.go index 3dcef6a604..1551ea33e8 100644 --- a/buildengine/discover.go +++ b/buildengine/discover.go @@ -72,7 +72,7 @@ func discoverModules(ctx context.Context, dirs ...string) ([]Module, error) { } } sort.Slice(out, func(i, j int) bool { - return out[i].Key() < out[j].Key() + return out[i].Config().Key < out[j].Config().Key }) return out, nil } diff --git a/buildengine/engine.go b/buildengine/engine.go index 8b6dcaabbf..c807e39a69 100644 --- a/buildengine/engine.go +++ b/buildengine/engine.go @@ -34,7 +34,7 @@ type schemaChange struct { // Engine for building a set of modules. type Engine struct { client ftlv1connect.ControllerServiceClient - projects map[string]Project + projects map[ProjectKey]Project dirs []string externalDirs []string controllerSchema *xsync.MapOf[string, *schema.Module] @@ -64,7 +64,7 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, dirs client: client, dirs: dirs, externalDirs: externalDirs, - projects: map[string]Project{}, + projects: map[ProjectKey]Project{}, controllerSchema: xsync.NewMapOf[string, *schema.Module](), schemaChanges: pubsub.New[schemaChange](), parallelism: runtime.NumCPU(), @@ -78,14 +78,14 @@ func New(ctx context.Context, client ftlv1connect.ControllerServiceClient, dirs projects, err := DiscoverProjects(ctx, dirs, externalDirs, true) if err != nil { - return nil, fmt.Errorf("Could not find projects: %w", err) + return nil, fmt.Errorf("could not find projects: %w", err) } for _, project := range projects { project, err := UpdateDependencies(ctx, project) if err != nil { return nil, err } - e.projects[project.Key()] = project + e.projects[project.Config().Key] = project } if client == nil { @@ -144,29 +144,29 @@ func (e *Engine) Close() error { // // If no modules are provided, the entire graph is returned. An error is returned if // any dependencies are missing. -func (e *Engine) Graph(projects ...string) (map[string][]string, error) { - out := map[string][]string{} +func (e *Engine) Graph(projects ...ProjectKey) (map[ProjectKey][]ProjectKey, error) { + out := map[ProjectKey][]ProjectKey{} if len(projects) == 0 { projects = maps.Keys(e.projects) } - for _, name := range projects { - if err := e.buildGraph(name, out); err != nil { + for _, key := range projects { + if err := e.buildGraph(key, out); err != nil { return nil, err } } return out, nil } -func (e *Engine) buildGraph(name string, out map[string][]string) error { - var deps []string - if project, ok := e.projects[name]; ok { - deps = project.Dependencies() - } else if sch, ok := e.controllerSchema.Load(name); ok { - deps = sch.Imports() +func (e *Engine) buildGraph(key ProjectKey, out map[ProjectKey][]ProjectKey) error { + var deps []ProjectKey + if project, ok := e.projects[ProjectKey(key)]; ok { + deps = project.Config().Dependencies + } else if sch, ok := e.controllerSchema.Load(string(key)); ok { + deps = ProjectKeysFromModuleNames(sch.Imports()) } else { - return fmt.Errorf("module %q not found", name) + return fmt.Errorf("module %q not found", key) } - out[name] = deps + out[key] = deps for _, dep := range deps { if err := e.buildGraph(dep, out); err != nil { return err @@ -239,29 +239,28 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration case event := <-watchEvents: switch event := event.(type) { case WatchEventProjectAdded: - if _, exists := e.projects[event.Project.Key()]; !exists { - e.projects[event.Project.Key()] = event.Project - err := e.buildAndDeploy(ctx, 1, true, event.Project.Key()) + config := event.Project.Config() + if _, exists := e.projects[config.Key]; !exists { + e.projects[config.Key] = event.Project + err := e.buildAndDeploy(ctx, 1, true, config.Key) if err != nil { - logger.Errorf(err, "deploy %s failed", event.Project.Key()) + logger.Errorf(err, "deploy %s failed", config.Key) } } case WatchEventProjectRemoved: - if _, ok := event.Project.(Module); ok { - err := teminateModuleDeployment(ctx, e.client, event.Project.Key()) + config := event.Project.Config() + if module, ok := event.Project.(Module); ok { + err := teminateModuleDeployment(ctx, e.client, module.Module) if err != nil { - logger.Errorf(err, "terminate %s failed", event.Project.Key()) + logger.Errorf(err, "terminate %s failed", module.Module) } } - delete(e.projects, event.Project.Key()) + delete(e.projects, config.Key) case WatchEventProjectChanged: - err := e.buildAndDeploy(ctx, 1, true, event.Project.Key()) + config := event.Project.Config() + err := e.buildAndDeploy(ctx, 1, true, config.Key) if err != nil { - if _, ok := event.Project.(Module); ok { - logger.Errorf(err, "deploy %s failed", event.Project.Key()) - } else { - logger.Errorf(err, "stub generation for %s failed", event.Project.Key()) - } + logger.Errorf(err, "build and deploy failed for %v: %w", event.Project, err) } } case change := <-schemaChanges: @@ -282,11 +281,11 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration moduleHashes[change.Name] = hash - dependantModules := e.getDependentModules(change.Name) - if len(dependantModules) > 0 { + dependentProjectKeys := e.getDependentProjectKeys(ProjectKeyForModuleName(change.Name)) + if len(dependentProjectKeys) > 0 { //TODO: inaccurate log message for ext libs - logger.Infof("%s's schema changed; redeploying %s", change.Name, strings.Join(dependantModules, ", ")) - err = e.buildAndDeploy(ctx, 1, true, dependantModules...) + logger.Infof("%s's schema changed; processing %s", change.Name, strings.Join([]string(StringsFromProjectKeys(dependentProjectKeys)), ", ")) + err = e.buildAndDeploy(ctx, 1, true, dependentProjectKeys...) if err != nil { logger.Errorf(err, "deploy %s failed", change.Name) } @@ -306,19 +305,19 @@ func computeModuleHash(module *schema.Module) ([]byte, error) { return hasher.Sum(nil), nil } -func (e *Engine) getDependentModules(projectName string) []string { - dependentProjects := map[string]bool{} +func (e *Engine) getDependentProjectKeys(key ProjectKey) []ProjectKey { + dependentProjectKeys := map[ProjectKey]bool{} for key, project := range e.projects { - for _, dep := range project.Dependencies() { - if dep == projectName { - dependentProjects[key] = true + for _, dep := range project.Config().Dependencies { + if dep == key { + dependentProjectKeys[key] = true } } } - return maps.Keys(dependentProjects) + return maps.Keys(dependentProjectKeys) } -func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDeployOnline bool, projects ...string) error { +func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDeployOnline bool, projects ...ProjectKey) error { if len(projects) == 0 { projects = maps.Keys(e.projects) } @@ -364,21 +363,20 @@ func (e *Engine) buildAndDeploy(ctx context.Context, replicas int32, waitForDepl } }) } - return wg.Wait() } type buildCallback func(ctx context.Context, project Project) error -func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, projects ...string) error { - mustBuild := map[string]bool{} +func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, projects ...ProjectKey) error { + mustBuild := map[ProjectKey]bool{} if len(projects) == 0 { projects = maps.Keys(e.projects) } - for _, name := range projects { - project, ok := e.projects[name] + for _, key := range projects { + project, ok := e.projects[key] if !ok { - return fmt.Errorf("project %q not found", project) + return fmt.Errorf("project %q not found", key) } // Update dependencies before building. var err error @@ -386,15 +384,15 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, if err != nil { return err } - e.projects[name] = project - mustBuild[name] = true + e.projects[key] = project + mustBuild[key] = true } graph, err := e.Graph(projects...) if err != nil { return err } - built := map[string]*schema.Module{ - "builtin": schema.Builtins(), + built := map[ProjectKey]*schema.Module{ + ProjectKeyForModuleName("builtin"): schema.Builtins(), } topology := TopologicalSort(graph) @@ -418,7 +416,7 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, return err case ExternalLibrary: if err := e.generateStubsForExternalLib(ctx, key, built); err != nil { - return fmt.Errorf("could not build stubs for external library %s: %w", key, err) + return fmt.Errorf("could not build stubs for %q: %w", key, err) } return nil default: @@ -433,7 +431,7 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, // Now this group is built, collect all the schemas. close(schemas) for sch := range schemas { - built[sch.Name] = sch + built[ProjectKeyForModuleName(sch.Name)] = sch } } @@ -441,18 +439,18 @@ func (e *Engine) buildWithCallback(ctx context.Context, callback buildCallback, } // Publish either the schema from the FTL controller, or from a local build. -func (e *Engine) mustSchema(ctx context.Context, name string, built map[string]*schema.Module, schemas chan<- *schema.Module) error { - if sch, ok := e.controllerSchema.Load(name); ok { +func (e *Engine) mustSchema(ctx context.Context, key ProjectKey, built map[ProjectKey]*schema.Module, schemas chan<- *schema.Module) error { + if sch, ok := e.controllerSchema.Load(string(key)); ok { schemas <- sch return nil } - return e.build(ctx, name, built, schemas) + return e.build(ctx, key, built, schemas) } // Build a module and publish its schema. // // Assumes that all dependencies have been built and are available in "built". -func (e *Engine) build(ctx context.Context, key string, built map[string]*schema.Module, schemas chan<- *schema.Module) error { +func (e *Engine) build(ctx context.Context, key ProjectKey, built map[ProjectKey]*schema.Module, schemas chan<- *schema.Module) error { project, ok := e.projects[key] if !ok { return fmt.Errorf("project %q not found", key) @@ -462,7 +460,7 @@ func (e *Engine) build(ctx context.Context, key string, built map[string]*schema return fmt.Errorf("can not build project %q as it is not a module", key) } - combined := map[string]*schema.Module{} + combined := map[ProjectKey]*schema.Module{} if err := e.gatherSchemas(built, project, combined); err != nil { return err } @@ -473,15 +471,15 @@ func (e *Engine) build(ctx context.Context, key string, built map[string]*schema return err } - moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(module.Dir(), module.DeployDir, module.Schema)) + moduleSchema, err := schema.ModuleFromProtoFile(filepath.Join(module.Dir, module.DeployDir, module.Schema)) if err != nil { - return fmt.Errorf("load schema %s: %w", key, err) + return fmt.Errorf("could not load schema for %s: %w", module, err) } schemas <- moduleSchema return nil } -func (e *Engine) generateStubsForExternalLib(ctx context.Context, key string, built map[string]*schema.Module) error { +func (e *Engine) generateStubsForExternalLib(ctx context.Context, key ProjectKey, built map[ProjectKey]*schema.Module) error { //TODO: support kotlin project, ok := e.projects[key] if !ok { @@ -494,9 +492,9 @@ func (e *Engine) generateStubsForExternalLib(ctx context.Context, key string, bu logger := log.FromContext(ctx) - goModFile, replacements, err := compile.GoModFileWithReplacements(filepath.Join(lib.Dir(), "go.mod")) + goModFile, replacements, err := compile.GoModFileWithReplacements(filepath.Join(lib.Dir, "go.mod")) if err != nil { - return fmt.Errorf("failed to propogate replacements %s: %w", lib.Dir(), err) + return fmt.Errorf("failed to propogate replacements for %s: %w", lib, err) } ftlVersion := "" @@ -504,37 +502,37 @@ func (e *Engine) generateStubsForExternalLib(ctx context.Context, key string, bu ftlVersion = ftl.Version } - combined := map[string]*schema.Module{} + combined := map[ProjectKey]*schema.Module{} if err := e.gatherSchemas(built, project, combined); err != nil { return err } sch := &schema.Schema{Modules: maps.Values(combined)} err = compile.GenerateExternalModules(compile.ExternalModuleContext{ - ModuleDir: lib.Dir(), + ModuleDir: lib.Dir, GoVersion: goModFile.Go.Version, FTLVersion: ftlVersion, Schema: sch, Replacements: replacements, }) if err != nil { - return fmt.Errorf("failed to generate external modules: %w", err) + return fmt.Errorf("failed to generate external modules for %s: %w", lib, err) } - logger.Infof("Generated stubs %v for %s", lib.Dependencies(), lib.Dir()) + logger.Infof("Generated stubs %v for %s", lib.Dependencies, lib.Dir) return nil } // Construct a combined schema for a module and its transitive dependencies. func (e *Engine) gatherSchemas( - moduleSchemas map[string]*schema.Module, + moduleSchemas map[ProjectKey]*schema.Module, project Project, - out map[string]*schema.Module, + out map[ProjectKey]*schema.Module, ) error { - latestModule, ok := e.projects[project.Key()] + latestModule, ok := e.projects[project.Config().Key] if !ok { latestModule = project } - for _, dep := range latestModule.Dependencies() { + for _, dep := range latestModule.Config().Dependencies { out[dep] = moduleSchemas[dep] if dep != "builtin" { depModule, ok := e.projects[dep] diff --git a/buildengine/filehash.go b/buildengine/filehash.go index 64bfbb5989..53a3e85d01 100644 --- a/buildengine/filehash.go +++ b/buildengine/filehash.go @@ -64,10 +64,12 @@ func CompareFileHashes(oldFiles, newFiles FileHashes) (FileChangeType, string, b // ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in // the given directory. func ComputeFileHashes(module Project) (FileHashes, error) { + config := module.Config() + fileHashes := make(FileHashes) - err := WalkDir(module.Dir(), func(srcPath string, entry fs.DirEntry) error { - for _, pattern := range module.Watch() { - relativePath, err := filepath.Rel(module.Dir(), srcPath) + err := WalkDir(config.Dir, func(srcPath string, entry fs.DirEntry) error { + for _, pattern := range config.Watch { + relativePath, err := filepath.Rel(config.Dir, srcPath) if err != nil { return err } diff --git a/buildengine/project.go b/buildengine/project.go index 7805f77585..79214d3178 100644 --- a/buildengine/project.go +++ b/buildengine/project.go @@ -10,96 +10,111 @@ import ( "github.com/TBD54566975/ftl/common/moduleconfig" ) +// Project models FTL modules and external libraries and is used to manage dependencies within the build engine type Project interface { - Dir() string - Key() string - Language() string - Watch() []string - Dependencies() []string - Kind() string - - CopyWithDependencies([]string) Project + Config() ProjectConfig + CopyWithDependencies([]ProjectKey) Project + String() string +} + +type ProjectConfig struct { + Key ProjectKey + Dir string + Language string + Watch []string + Dependencies []ProjectKey } var _ = (Project)(Module{}) var _ = (Project)(ExternalLibrary{}) +// Module represents an FTL module in the build engine type Module struct { moduleconfig.ModuleConfig - Deps []string -} - -func (m Module) Dir() string { - return m.ModuleConfig.Dir -} - -func (m Module) Key() string { - return m.ModuleConfig.Module -} - -func (m Module) Language() string { - return m.ModuleConfig.Language -} - -func (m Module) Watch() []string { - return m.ModuleConfig.Watch + Dependencies []string } -func (m Module) Dependencies() []string { - return m.Deps +func (m Module) Config() ProjectConfig { + return ProjectConfig{ + Key: ProjectKey(m.ModuleConfig.Module), + Dir: m.ModuleConfig.Dir, + Language: m.ModuleConfig.Language, + Watch: m.ModuleConfig.Watch, + Dependencies: ProjectKeysFromModuleNames(m.Dependencies), + } } -func (m Module) CopyWithDependencies(dependencies []string) Project { +func (m Module) CopyWithDependencies(dependencies []ProjectKey) Project { module := reflect.DeepCopy(m) - module.Deps = dependencies + module.Dependencies = StringsFromProjectKeys(dependencies) return Project(module) } -func (m Module) Kind() string { - return "module" +func (m Module) String() string { + return "module:" + m.ModuleConfig.Module } +// ExternalLibrary represents a library that makes use of FTL modules, but is not itself an FTL module type ExternalLibrary struct { - Directory string - Lang string - Deps []string -} - -func (e ExternalLibrary) Dir() string { - return e.Directory -} - -func (e ExternalLibrary) Key() string { - return e.Dir() -} - -func (e ExternalLibrary) Language() string { - return e.Lang + Dir string + Language string + Dependencies []string } -func (e ExternalLibrary) Watch() []string { - switch e.Lang { +func (e ExternalLibrary) Config() ProjectConfig { + var watch []string + switch e.Language { case "go": - return []string{"**/*.go", "go.mod", "go.sum"} + watch = []string{"**/*.go", "go.mod", "go.sum"} case "kotlin": - return []string{"pom.xml", "src/**", "target/generated-sources"} + watch = []string{"pom.xml", "src/**", "target/generated-sources"} default: - panic(fmt.Sprintf("unknown language %T", e.Lang)) + panic(fmt.Sprintf("unknown language %T", e.Language)) } -} -func (e ExternalLibrary) Dependencies() []string { - return e.Deps + return ProjectConfig{ + Key: ProjectKey("lib:" + e.Dir), + Dir: e.Dir, + Language: e.Language, + Watch: watch, + Dependencies: ProjectKeysFromModuleNames(e.Dependencies), + } } -func (e ExternalLibrary) CopyWithDependencies(dependencies []string) Project { +func (e ExternalLibrary) CopyWithDependencies(dependencies []ProjectKey) Project { lib := reflect.DeepCopy(e) - lib.Deps = dependencies + lib.Dependencies = StringsFromProjectKeys(dependencies) return Project(lib) } -func (e ExternalLibrary) Kind() string { - return "library" +func (e ExternalLibrary) String() string { + return "library:" + e.Dir +} + +// Key is a unique identifier for the project (ie: a module name or a library path) +// It is used to: +// - build the dependency graph +// - map changes in the file system to the project +type ProjectKey string + +func ProjectKeyForModuleName(name string) ProjectKey { + return ProjectKey(name) +} + +func StringsFromProjectKeys(keys []ProjectKey) []string { + strs := make([]string, len(keys)) + for i, key := range keys { + strs[i] = string(key) + } + return strs +} + +func ProjectKeysFromModuleNames(names []string) []ProjectKey { + keys := make([]ProjectKey, len(names)) + for i, str := range names { + keys[i] = ProjectKey(str) + } + return keys } // LoadModule loads a module from the given directory. @@ -114,23 +129,23 @@ func LoadModule(dir string) (Module, error) { func LoadExternalLibrary(dir string) (ExternalLibrary, error) { lib := ExternalLibrary{ - Directory: dir, + Dir: dir, } goModPath := filepath.Join(dir, "go.mod") pomPath := filepath.Join(dir, "pom.xml") if _, err := os.Stat(goModPath); err == nil { - lib.Lang = "go" + lib.Language = "go" } else if !os.IsNotExist(err) { return ExternalLibrary{}, err } else { if _, err = os.Stat(pomPath); err == nil { - lib.Lang = "kotlin" + lib.Language = "kotlin" } else if !os.IsNotExist(err) { return ExternalLibrary{}, err } } - if lib.Lang == "" { + if lib.Language == "" { return ExternalLibrary{}, fmt.Errorf("could not autodetect language: no go.mod or pom.xml found in %s", dir) } diff --git a/buildengine/topological.go b/buildengine/topological.go index 70486338be..20ac73179f 100644 --- a/buildengine/topological.go +++ b/buildengine/topological.go @@ -8,23 +8,23 @@ import ( // TopologicalSort returns a sequence of groups of modules in topological order // that may be built in parallel. -func TopologicalSort(graph map[string][]string) [][]string { - modulesByName := map[string]bool{} +func TopologicalSort(graph map[ProjectKey][]ProjectKey) [][]ProjectKey { + modulesByKey := map[ProjectKey]bool{} for module := range graph { - modulesByName[module] = true + modulesByKey[module] = true } // Order of modules to build. // Each element is a list of modules that can be built in parallel. - groups := [][]string{} + groups := [][]ProjectKey{} // Modules that have already been "built" - built := map[string]bool{"builtin": true} + built := map[ProjectKey]bool{"builtin": true} - for len(modulesByName) > 0 { + for len(modulesByKey) > 0 { // Current group of modules that can be built in parallel. - group := map[string]bool{} + group := map[ProjectKey]bool{} nextModule: - for module := range modulesByName { + for module := range modulesByKey { // Check that all dependencies have been built. for _, dep := range graph[module] { if !built[dep] { @@ -32,10 +32,13 @@ func TopologicalSort(graph map[string][]string) [][]string { } } group[module] = true - delete(modulesByName, module) + delete(modulesByKey, module) } orderedGroup := maps.Keys(group) - sort.Strings(orderedGroup) + + sort.Slice(orderedGroup, func(i, j int) bool { + return orderedGroup[i] < orderedGroup[j] + }) for _, module := range orderedGroup { built[module] = true } diff --git a/buildengine/watch.go b/buildengine/watch.go index 7b077f56e9..ce735c3706 100644 --- a/buildengine/watch.go +++ b/buildengine/watch.go @@ -57,28 +57,26 @@ func Watch(ctx context.Context, period time.Duration, dirs []string, externalLib projects, _ := DiscoverProjects(ctx, dirs, externalLibDirs, false) projectsByDir := maps.FromSlice(projects, func(project Project) (string, Project) { - return project.Dir(), project + return project.Config().Dir, project }) // Trigger events for removed projects. for _, existingProject := range existingProjects { - if _, haveProject := projectsByDir[existingProject.Project.Dir()]; !haveProject { - if _, ok := existingProject.Project.(Module); ok { - logger.Debugf("%s %s removed: %s", existingProject.Project.Kind(), existingProject.Project.Key(), existingProject.Project.Dir()) - } else { - logger.Debugf("%s removed: %s", existingProject.Project.Kind(), existingProject.Project.Dir()) - } + existingConfig := existingProject.Project.Config() + if _, haveProject := projectsByDir[existingConfig.Dir]; !haveProject { + logger.Debugf("removed %s", existingProject.Project) topic.Publish(WatchEventProjectRemoved{Project: existingProject.Project}) - delete(existingProjects, existingProject.Project.Dir()) + delete(existingProjects, existingConfig.Dir) } } // Compare the projects to the existing projects. for _, project := range projectsByDir { - existingProject, haveExistingProject := existingProjects[project.Dir()] + config := project.Config() + existingProject, haveExistingProject := existingProjects[config.Dir] hashes, err := ComputeFileHashes(project) if err != nil { - logger.Tracef("error computing file hashes for %s: %v", project.Dir(), err) + logger.Tracef("error computing file hashes for %s: %v", config.Dir, err) continue } @@ -87,19 +85,14 @@ func Watch(ctx context.Context, period time.Duration, dirs []string, externalLib if equal { continue } - logger.Debugf("%s %s changed: %c%s", project.Kind(), project.Key(), changeType, path) + logger.Debugf("changed %s: %c%s", project, changeType, path) topic.Publish(WatchEventProjectChanged{Project: existingProject.Project, Change: changeType, Path: path}) - existingProjects[project.Dir()] = projectHashes{Hashes: hashes, Project: existingProject.Project} + existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: existingProject.Project} continue } - if _, ok := project.(Module); ok { - logger.Debugf("%s %s added: %s", project.Kind(), project.Key(), project.Dir()) - } else { - logger.Debugf("%s added: %s", project.Kind(), project.Dir()) - } - + logger.Debugf("added %s", project) topic.Publish(WatchEventProjectAdded{Project: project}) - existingProjects[project.Dir()] = projectHashes{Hashes: hashes, Project: project} + existingProjects[config.Dir] = projectHashes{Hashes: hashes, Project: project} } } }()