feat: start work on the FTL build engine package (#965)

I've moved a bunch of code out of the cmd/ftl package, moved
ModuleConfig and related functions into the package, and started
implementing some of the more foundational aspects of the build engine.
Specifically:

1. A `DiscoverModules(dirs...)` function that recursively discovers FTL
modules and returns their ModuleConfig.
2. Implemented functions to quickly extract FTL dependencies from Go FTL
modules (Kotlin is not implemented yet).
3. Implemented a `BuildOrder(modules...)` function that returns groups
of modules in topological order that can be built in parallel.

backend/controller/scaling/localscaling/devel.go

@@ -16,11 +16,11 @@ var templateDirOnce sync.Once
 
 func templateDir(ctx context.Context) string {
 	templateDirOnce.Do(func() {
-		cmd := exec.Command(ctx, log.Debug, internal.FTLSourceRoot(), "bit", "build/template/ftl/jars/ftl-runtime.jar")
+		cmd := exec.Command(ctx, log.Debug, internal.GitRoot(""), "bit", "build/template/ftl/jars/ftl-runtime.jar")
 		err := cmd.Run()
 		if err != nil {
 			panic(err)
 		}
 	})
-	return filepath.Join(internal.FTLSourceRoot(), "build/template")
+	return filepath.Join(internal.GitRoot(""), "build/template")
 }

buildengine/build.go

@@ -0,0 +1,12 @@
+package buildengine
+
+import (
+	"context"
+
+	"github.com/TBD54566975/ftl/backend/schema"
+)
+
+// Build a module in the given directory given the schema and module config.
+func Build(ctx context.Context, schema *schema.Schema, config ModuleConfig) error {
+	panic("not implemented")
+}

buildengine/deps.go

@@ -0,0 +1,98 @@
+package buildengine
+
+import (
+	"fmt"
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.design/x/reflect"
+	"golang.org/x/exp/maps"
+)
+
+// UpdateAllDependencies returns deep copies of all modules with updated dependencies.
+func UpdateAllDependencies(modules []ModuleConfig) ([]ModuleConfig, error) {
+	modulesByName := map[string]ModuleConfig{}
+	for _, module := range modules {
+		modulesByName[module.Module] = module
+	}
+	out := []ModuleConfig{}
+	for _, module := range modules {
+		updated, err := UpdateDependencies(module)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, updated)
+	}
+	return out, nil
+}
+
+// UpdateDependencies returns a deep copy of ModuleConfig with updated dependencies.
+func UpdateDependencies(config ModuleConfig) (ModuleConfig, error) {
+	dependencies, err := extractDependencies(config)
+	if err != nil {
+		return ModuleConfig{}, err
+	}
+	out := reflect.DeepCopy(config)
+	out.Dependencies = dependencies
+	return out, nil
+}
+
+func extractDependencies(config ModuleConfig) ([]string, error) {
+	switch config.Language {
+	case "go":
+		return extractGoFTLImports(config.Module, config.Dir)
+
+	case "kotlin":
+		return extractKotlinFTLImports(config.Dir)
+
+	default:
+		return nil, fmt.Errorf("unsupported language: %s", config.Language)
+	}
+}
+
+func extractGoFTLImports(self, dir string) ([]string, error) {
+	dependencies := map[string]bool{}
+	fset := token.NewFileSet()
+	err := WalkDir(dir, func(path string, d fs.DirEntry) error {
+		if !d.IsDir() {
+			return nil
+		}
+		pkgs, err := parser.ParseDir(fset, path, nil, parser.ImportsOnly)
+		if pkgs == nil {
+			return err
+		}
+		for _, pkg := range pkgs {
+			for _, file := range pkg.Files {
+				for _, imp := range file.Imports {
+					path, err := strconv.Unquote(imp.Path.Value)
+					if err != nil {
+						continue
+					}
+					if !strings.HasPrefix(path, "ftl/") {
+						continue
+					}
+					module := strings.Split(strings.TrimPrefix(path, "ftl/"), "/")[0]
+					if module == self {
+						continue
+					}
+					dependencies[module] = true
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("%s: failed to extract dependencies from Go module: %w", self, err)
+	}
+	modules := maps.Keys(dependencies)
+	sort.Strings(modules)
+	return modules, nil
+}
+
+func extractKotlinFTLImports(dir string) ([]string, error) {
+	panic("not implemented")
+}

buildengine/deps_test.go

@@ -0,0 +1,13 @@
+package buildengine
+
+import (
+	"testing"
+
+	"github.com/alecthomas/assert/v2"
+)
+
+func TestExtractDepsGo(t *testing.T) {
+	deps, err := extractGoFTLImports("test", "testdata/modules/alpha")
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"another", "other"}, deps)
+}

buildengine/discover.go

@@ -0,0 +1,43 @@
+package buildengine
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// DiscoverModules recursively loads all modules under the given directories.
+//
+// If no directories are provided, the current working directory is used.
+func DiscoverModules(dirs ...string) ([]ModuleConfig, error) {
+	if len(dirs) == 0 {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		dirs = []string{cwd}
+	}
+	out := []ModuleConfig{}
+	for _, dir := range dirs {
+		err := WalkDir(dir, func(path string, d fs.DirEntry) error {
+			if filepath.Base(path) != "ftl.toml" {
+				return nil
+			}
+			moduleDir := filepath.Dir(path)
+			config, err := LoadModuleConfig(moduleDir)
+			if err != nil {
+				return err
+			}
+			out = append(out, config)
+			return ErrSkip
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Module < out[j].Module
+	})
+	return out, nil
+}

buildengine/discover_test.go

@@ -0,0 +1,45 @@
+package buildengine
+
+import (
+	"testing"
+
+	"github.com/alecthomas/assert/v2"
+)
+
+func TestDiscoverModules(t *testing.T) {
+	modules, err := DiscoverModules("testdata/modules")
+	assert.NoError(t, err)
+	expected := []ModuleConfig{
+		{
+			Dir:       "testdata/modules/alpha",
+			Language:  "go",
+			Realm:     "home",
+			Module:    "alpha",
+			Deploy:    []string{"main"},
+			DeployDir: "_ftl",
+			Schema:    "schema.pb",
+			Watch:     []string{"**/*.go", "go.mod", "go.sum"},
+		},
+		{
+			Dir:       "testdata/modules/another",
+			Language:  "go",
+			Realm:     "home",
+			Module:    "another",
+			Deploy:    []string{"main"},
+			DeployDir: "_ftl",
+			Schema:    "schema.pb",
+			Watch:     []string{"**/*.go", "go.mod", "go.sum"},
+		},
+		{
+			Dir:       "testdata/modules/other",
+			Language:  "go",
+			Realm:     "home",
+			Module:    "other",
+			Deploy:    []string{"main"},
+			DeployDir: "_ftl",
+			Schema:    "schema.pb",
+			Watch:     []string{"**/*.go", "go.mod", "go.sum"},
+		},
+	}
+	assert.Equal(t, expected, modules)
+}

buildengine/docs.go

@@ -0,0 +1,2 @@
+// Package buildengine provides a framework for building FTL modules.
+package buildengine

buildengine/filehash.go

@@ -0,0 +1,88 @@
+package buildengine
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/bmatcuk/doublestar/v4"
+)
+
+// CompareFileHashes compares the hashes of the files in the oldFiles and newFiles maps.
+//
+// Returns true if the hashes are equal, false otherwise.
+//
+// If false, the returned string will be the file that caused the difference,
+// prefixed with "+" if it's a new file, or "-" if it's a removed file.
+func CompareFileHashes(oldFiles, newFiles map[string][]byte) (string, bool) {
+	for key, hash1 := range oldFiles {
+		hash2, exists := newFiles[key]
+		if !exists {
+			return "-" + key, false
+		}
+		if !bytes.Equal(hash1, hash2) {
+			return key, false
+		}
+	}
+
+	for key := range newFiles {
+		if _, exists := oldFiles[key]; !exists {
+			return "+" + key, false
+		}
+	}
+
+	return "", true
+}
+
+// ComputeFileHashes computes the SHA256 hash of all (non-git-ignored) files in
+// the given directory.
+func ComputeFileHashes(dir string) (map[string][]byte, error) {
+	config, err := LoadModuleConfig(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	fileHashes := make(map[string][]byte)
+	err = WalkDir(dir, func(srcPath string, entry fs.DirEntry) error {
+		for _, pattern := range config.Watch {
+			relativePath, err := filepath.Rel(dir, srcPath)
+			if err != nil {
+				return err
+			}
+
+			match, err := doublestar.PathMatch(pattern, relativePath)
+			if err != nil {
+				return err
+			}
+
+			if match && !entry.IsDir() {
+				file, err := os.Open(srcPath)
+				if err != nil {
+					return err
+				}
+
+				hasher := sha256.New()
+				if _, err := io.Copy(hasher, file); err != nil {
+					_ = file.Close()
+					return err
+				}
+
+				fileHashes[srcPath] = hasher.Sum(nil)
+
+				if err := file.Close(); err != nil {
+					return err
+				}
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return fileHashes, err
+}

internal/moduleconfig/config.go → buildengine/moduleconfig.go

@@ -1,4 +1,4 @@
-package moduleconfig
+package buildengine
 
 import (
 	"fmt"
@@ -8,32 +8,34 @@ import (
 	"github.com/BurntSushi/toml"
 )
 
-// GoConfig is language-specific configuration for Go modules.
-type GoConfig struct {
-}
+// ModuleGoConfig is language-specific configuration for Go modules.
+type ModuleGoConfig struct{}
 
-// KotlinConfig is language-specific configuration for Kotlin modules.
-type KotlinConfig struct {
-}
+// ModuleKotlinConfig is language-specific configuration for Kotlin modules.
+type ModuleKotlinConfig struct{}
 
 // ModuleConfig is the configuration for an FTL module.
 //
 // Module config files are currently TOML.
 type ModuleConfig struct {
+	Dir          string   `toml:"-"` // Populated by LoadConfig.
+	Dependencies []string `toml:"-"` // Populated by BuildOrder.
+
 	Language  string   `toml:"language"`
+	Realm     string   `toml:"realm"`
 	Module    string   `toml:"module"`
 	Build     string   `toml:"build"`
 	Deploy    []string `toml:"deploy"`
 	DeployDir string   `toml:"deploy-dir"`
 	Schema    string   `toml:"schema"`
 	Watch     []string `toml:"watch"`
 
-	Go     GoConfig     `toml:"go,optional"`
-	Kotlin KotlinConfig `toml:"kotlin,optional"`
+	Go     ModuleGoConfig     `toml:"go,optional"`
+	Kotlin ModuleKotlinConfig `toml:"kotlin,optional"`
 }
 
-// LoadConfig from a directory.
-func LoadConfig(dir string) (ModuleConfig, error) {
+// LoadModuleConfig from a directory.
+func LoadModuleConfig(dir string) (ModuleConfig, error) {
 	path := filepath.Join(dir, "ftl.toml")
 	config := ModuleConfig{}
 	_, err := toml.DecodeFile(path, &config)
@@ -43,10 +45,14 @@ func LoadConfig(dir string) (ModuleConfig, error) {
 	if err := setConfigDefaults(dir, &config); err != nil {
 		return config, fmt.Errorf("%s: %w", path, err)
 	}
+	config.Dir = dir
 	return config, nil
 }
 
 func setConfigDefaults(moduleDir string, config *ModuleConfig) error {
+	if config.Realm == "" {
+		config.Realm = "home"
+	}
 	if config.Schema == "" {
 		config.Schema = "schema.pb"
 	}

buildengine/testdata/modules/alpha/ftl.toml

@@ -0,0 +1,2 @@
+module = "alpha"
+language = "go"