refactor: external libs are models with buildengine.Module

buildengine/build.go

@@ -3,43 +3,152 @@ package buildengine
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/moduleconfig"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
-// A Module is a ModuleConfig with its dependencies populated.
+// A Module is a ModuleConfig or ExternalLibrary with its dependencies populated.
 type Module struct {
-	moduleconfig.ModuleConfig
+	internal     interface{}
 	Dependencies []string
 }
 
+type ExternalLibrary struct {
+	Dir      string
+	Language string
+}
+
+func (m Module) Key() string {
+	switch module := m.internal.(type) {
+	case moduleconfig.ModuleConfig:
+		return module.Module
+	case ExternalLibrary:
+		return module.Dir
+	default:
+		panic(fmt.Sprintf("unknown internal type %T", m.internal))
+	}
+}
+
+func (m Module) Language() string {
+	switch module := m.internal.(type) {
+	case moduleconfig.ModuleConfig:
+		return module.Language
+	case ExternalLibrary:
+		return module.Language
+	default:
+		panic(fmt.Sprintf("unknown internal type %T", m.internal))
+	}
+}
+
+func (m Module) Dir() string {
+	switch module := m.internal.(type) {
+	case moduleconfig.ModuleConfig:
+		return module.Dir
+	case ExternalLibrary:
+		return module.Dir
+	default:
+		panic(fmt.Sprintf("unknown internal type %T", m.internal))
+	}
+}
+
+func (m Module) Watch() []string {
+	switch module := m.internal.(type) {
+	case moduleconfig.ModuleConfig:
+		return module.Watch
+	case ExternalLibrary:
+		switch module.Language {
+		case "go":
+			return []string{"**/*.go", "go.mod", "go.sum"}
+		case "kotlin":
+			return []string{"pom.xml", "src/**", "target/generated-sources"}
+		default:
+			panic(fmt.Sprintf("unknown language %T", m.Language()))
+		}
+	default:
+		panic(fmt.Sprintf("unknown internal type %T", m.internal))
+	}
+}
+
+func (m Module) Kind() string {
+	switch m.internal.(type) {
+	case moduleconfig.ModuleConfig:
+		return "module"
+	case ExternalLibrary:
+		return "library"
+	default:
+		panic(fmt.Sprintf("unknown internal type %T", m.internal))
+	}
+}
+
+func (m Module) ModuleConfig() (moduleconfig.ModuleConfig, bool) {
+	config, ok := m.internal.(moduleconfig.ModuleConfig)
+	return config, ok
+}
+
+func (m Module) ExternalLibrary() (ExternalLibrary, bool) {
+	lib, ok := m.internal.(ExternalLibrary)
+	return lib, ok
+}
+
 // LoadModule loads a module from the given directory.
-//
-// A [Module] includes the module configuration as well as its dependencies
-// extracted from source code.
 func LoadModule(ctx context.Context, dir string) (Module, error) {
 	config, err := moduleconfig.LoadModuleConfig(dir)
 	if err != nil {
 		return Module{}, err
 	}
-	return UpdateDependencies(ctx, config)
+	module := Module{internal: config}
+	return module, nil
+}
+
+func LoadExternalLibrary(ctx context.Context, dir string) (Module, error) {
+	lib := ExternalLibrary{
+		Dir: dir,
+	}
+
+	goModPath := filepath.Join(dir, "go.mod")
+	pomPath := filepath.Join(dir, "pom.xml")
+	if _, err := os.Stat(goModPath); err == nil {
+		lib.Language = "go"
+	} else if !os.IsNotExist(err) {
+		return Module{}, err
+	} else {
+		if _, err = os.Stat(pomPath); err == nil {
+			lib.Language = "kotlin"
+		} else if !os.IsNotExist(err) {
+			return Module{}, err
+		}
+	}
+	if lib.Language == "" {
+		return Module{}, fmt.Errorf("could not autodetect language: no go.mod or pom.xml found in %s", dir)
+	}
+
+	module := Module{
+		internal: lib,
+	}
+	return module, nil
 }
 
 // Build a module in the given directory given the schema and module config.
 func Build(ctx context.Context, sch *schema.Schema, module Module) error {
-	logger := log.FromContext(ctx).Scope(module.Module)
+	config, ok := module.ModuleConfig()
+	if !ok {
+		return fmt.Errorf("cannot build module without module config: %q", module.Key())
+	}
+	logger := log.FromContext(ctx).Scope(config.Module)
 	ctx = log.ContextWithLogger(ctx, logger)
 	logger.Infof("Building module")
-	switch module.Language {
+	switch config.Language {
 	case "go":
 		return buildGo(ctx, sch, module)
 
 	case "kotlin":
 		return buildKotlin(ctx, sch, module)
 
 	default:
-		return fmt.Errorf("unknown language %q", module.Language)
+		return fmt.Errorf("unknown language %q", config.Language)
 	}
 }

buildengine/build_go.go

@@ -9,8 +9,12 @@ import (
 )
 
 func buildGo(ctx context.Context, sch *schema.Schema, module Module) error {
-	if err := compile.Build(ctx, module.Dir, sch); err != nil {
-		return fmt.Errorf("failed to build module %s: %w", module.Module, err)
+	moduleConfig, ok := module.ModuleConfig()
+	if !ok {
+		return fmt.Errorf("module %s is not a FTL module", module.Key())
+	}
+	if err := compile.Build(ctx, moduleConfig.Dir, sch); err != nil {
+		return fmt.Errorf("failed to build module %s: %w", moduleConfig.Module, err)
 	}
 	return nil
 }

buildengine/build_kotlin.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/TBD54566975/ftl"
 	"github.com/TBD54566975/ftl/backend/schema"
+	"github.com/TBD54566975/ftl/common/moduleconfig"
 	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/exec"
 	"github.com/TBD54566975/ftl/internal/log"
@@ -28,9 +29,13 @@ type externalModuleContext struct {
 }
 
 func (e externalModuleContext) ExternalModules() []*schema.Module {
+	name := ""
+	if config, ok := e.module.ModuleConfig(); ok {
+		name = config.Module
+	}
 	modules := make([]*schema.Module, 0, len(e.Modules))
 	for _, module := range e.Modules {
-		if module.Name == e.module.Module {
+		if module.Name == name {
 			continue
 		}
 		modules = append(modules, module)
@@ -39,23 +44,28 @@ func (e externalModuleContext) ExternalModules() []*schema.Module {
 }
 
 func buildKotlin(ctx context.Context, sch *schema.Schema, module Module) error {
+	config, ok := module.ModuleConfig()
+	if !ok {
+		return fmt.Errorf("module %s is not a FTL module", module.Key())
+	}
+
 	logger := log.FromContext(ctx)
-	if err := SetPOMProperties(ctx, module.Dir); err != nil {
+	if err := SetPOMProperties(ctx, config.Dir); err != nil {
 		return fmt.Errorf("unable to update ftl.version in %s: %w", module.Dir, err)
 	}
 
 	if err := generateExternalModules(ctx, module, sch); err != nil {
-		return fmt.Errorf("unable to generate external modules for %s: %w", module.Module, err)
+		return fmt.Errorf("unable to generate external modules for %s: %w", config.Module, err)
 	}
 
-	if err := prepareFTLRoot(module); err != nil {
-		return fmt.Errorf("unable to prepare FTL root for %s: %w", module.Module, err)
+	if err := prepareFTLRoot(config); err != nil {
+		return fmt.Errorf("unable to prepare FTL root for %s: %w", config.Module, err)
 	}
 
-	logger.Debugf("Using build command '%s'", module.Build)
-	err := exec.Command(ctx, log.Debug, module.Dir, "bash", "-c", module.Build).RunBuffered(ctx)
+	logger.Debugf("Using build command '%s'", config.Build)
+	err := exec.Command(ctx, log.Debug, config.Dir, "bash", "-c", config.Build).RunBuffered(ctx)
 	if err != nil {
-		return fmt.Errorf("failed to build module %s: %w", module.Module, err)
+		return fmt.Errorf("failed to build module %s: %w", config.Module, err)
 	}
 
 	return nil
@@ -92,8 +102,8 @@ func SetPOMProperties(ctx context.Context, baseDir string) error {
 	return tree.WriteToFile(pomFile)
 }
 
-func prepareFTLRoot(module Module) error {
-	buildDir := module.AbsDeployDir()
+func prepareFTLRoot(config moduleconfig.ModuleConfig) error {
+	buildDir := config.AbsDeployDir()
 	if err := os.MkdirAll(buildDir, 0700); err != nil {
 		return err
 	}
@@ -107,7 +117,7 @@ SchemaExtractorRuleSet:
 
 	detektYmlPath := filepath.Join(buildDir, "detekt.yml")
 	if err := os.WriteFile(detektYmlPath, []byte(fileContent), 0600); err != nil {
-		return fmt.Errorf("unable to configure detekt for %s: %w", module.Module, err)
+		return fmt.Errorf("unable to configure detekt for %s: %w", config.Module, err)
 	}
 
 	mainFilePath := filepath.Join(buildDir, "main")
@@ -116,21 +126,20 @@ SchemaExtractorRuleSet:
 exec java -cp "classes:$(cat classpath.txt)" xyz.block.ftl.main.MainKt
 `
 	if err := os.WriteFile(mainFilePath, []byte(mainFile), 0700); err != nil { //nolint:gosec
-		return fmt.Errorf("unable to configure main executable for %s: %w", module.Module, err)
+		return fmt.Errorf("unable to configure main executable for %s: %w", config.Module, err)
 	}
 	return nil
 }
 
 func generateExternalModules(ctx context.Context, module Module, sch *schema.Schema) error {
 	logger := log.FromContext(ctx)
-	config := module.ModuleConfig
 	funcs := maps.Clone(scaffoldFuncs)
 
 	// Wipe the modules directory to ensure we don't have any stale modules.
-	_ = os.RemoveAll(filepath.Join(config.Dir, "target", "generated-sources", "ftl"))
+	_ = os.RemoveAll(filepath.Join(module.Dir(), "target", "generated-sources", "ftl"))
 
 	logger.Debugf("Generating external modules")
-	return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), config.Dir, externalModuleContext{
+	return internal.ScaffoldZip(kotlinruntime.ExternalModuleTemplates(), module.Dir(), externalModuleContext{
 		module: module,
 		Schema: sch,
 	}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs))

buildengine/deploy.go

@@ -36,12 +36,17 @@ type DeployClient interface {
 
 // Deploy a module to the FTL controller with the given number of replicas. Optionally wait for the deployment to become ready.
 func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnline bool, client DeployClient) error {
-	logger := log.FromContext(ctx).Scope(module.Module)
+	moduleConfig, ok := module.ModuleConfig()
+	if !ok {
+		return fmt.Errorf("can not deploy non-FTL module %s", module.Key())
+	}
+
+	logger := log.FromContext(ctx).Scope(moduleConfig.Module)
 	ctx = log.ContextWithLogger(ctx, logger)
 	logger.Infof("Deploying module")
 
-	deployDir := module.AbsDeployDir()
-	files, err := findFiles(deployDir, module.Deploy)
+	deployDir := moduleConfig.AbsDeployDir()
+	files, err := findFiles(deployDir, moduleConfig.Deploy)
 	if err != nil {
 		logger.Errorf(err, "failed to find files in %s", deployDir)
 		return err
@@ -57,9 +62,9 @@ func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnl
 		return err
 	}
 
-	moduleSchema, err := loadProtoSchema(deployDir, module.ModuleConfig, replicas)
+	moduleSchema, err := loadProtoSchema(deployDir, moduleConfig, replicas)
 	if err != nil {
-		return fmt.Errorf("failed to load protobuf schema from %q: %w", module.ModuleConfig.Schema, err)
+		return fmt.Errorf("failed to load protobuf schema from %q: %w", moduleConfig.Schema, err)
 	}
 
 	logger.Debugf("Uploading %d/%d files", len(gadResp.Msg.MissingDigests), len(files))

buildengine/deps.go

@@ -17,12 +17,11 @@ import (
 	"golang.design/x/reflect"
 	"golang.org/x/exp/maps"
 
-	"github.com/TBD54566975/ftl/common/moduleconfig"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
 // UpdateAllDependencies calls UpdateDependencies on each module in the list.
-func UpdateAllDependencies(ctx context.Context, modules ...moduleconfig.ModuleConfig) ([]Module, error) {
+func UpdateAllDependencies(ctx context.Context, modules ...Module) ([]Module, error) {
 	out := []Module{}
 	for _, module := range modules {
 		updated, err := UpdateDependencies(ctx, module)
@@ -34,12 +33,12 @@ func UpdateAllDependencies(ctx context.Context, modules ...moduleconfig.ModuleCo
 	return out, nil
 }
 
-// UpdateDependencies finds the dependencies for an FTL module and returns a
+// UpdateDependencies finds the dependencies for a module and returns a
 // Module with those dependencies populated.
-func UpdateDependencies(ctx context.Context, config moduleconfig.ModuleConfig) (Module, error) {
+func UpdateDependencies(ctx context.Context, module Module) (Module, error) {
 	logger := log.FromContext(ctx)
-	logger.Debugf("Extracting dependencies for module %s", config.Module)
-	dependencies, err := extractDependencies(config)
+	logger.Debugf("Extracting dependencies for module %s", module.Key())
+	dependencies, err := extractDependencies(module)
 	if err != nil {
 		return Module{}, err
 	}
@@ -53,20 +52,26 @@ func UpdateDependencies(ctx context.Context, config moduleconfig.ModuleConfig) (
 	if !containsBuiltin {
 		dependencies = append(dependencies, "builtin")
 	}
-	out := reflect.DeepCopy(config)
-	return Module{ModuleConfig: out, Dependencies: dependencies}, nil
+
+	out := reflect.DeepCopy(module)
+	out.Dependencies = dependencies
+	return out, nil
 }
 
-func extractDependencies(config moduleconfig.ModuleConfig) ([]string, error) {
-	switch config.Language {
+func extractDependencies(module Module) ([]string, error) {
+	name := ""
+	if config, ok := module.ModuleConfig(); ok {
+		name = config.Module
+	}
+	switch module.Language() {
 	case "go":
-		return extractGoFTLImports(config.Module, config.Dir)
+		return extractGoFTLImports(name, module.Dir())
 
 	case "kotlin":
-		return extractKotlinFTLImports(config.Module, config.Dir)
+		return extractKotlinFTLImports(name, module.Dir())
 
 	default:
-		return nil, fmt.Errorf("unsupported language: %s", config.Language)
+		return nil, fmt.Errorf("unsupported language: %s", module.Language())
 	}
 }