feat: implement the core build engine

This brings all of the existing buildengine utilities together into a
single system that unifies schema changes from the FTL controller with
local modules, supporting both online and completely offline builds.

Builds are run in parallel, with dependencies preferencing schemas from
other local modules first, then falling back on schemas from the FTL
controller.

.golangci.yml

@@ -132,3 +132,5 @@ issues:
     - 'should have comment \(or a comment on this block\) or be unexported'
     - caseOrder
     - unused-parameter
+    - "^loopclosure:"
+    - 'shadow: declaration of "ctx" shadows declaration at'

backend/controller/dal/dal_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -22,8 +21,7 @@ import (
 
 //nolint:maintidx
 func TestDAL(t *testing.T) {
-	logger := log.Configure(os.Stderr, log.Config{Level: log.Debug})
-	ctx := log.ContextWithLogger(context.Background(), logger)
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	conn := sqltest.OpenForTesting(ctx, t)
 	dal, err := New(ctx, conn)
 	assert.NoError(t, err)

backend/controller/scheduledtask/scheduledtask_test.go

@@ -2,7 +2,6 @@ package scheduledtask
 
 import (
 	"context"
-	"os"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -19,7 +18,7 @@ import (
 
 func TestCron(t *testing.T) {
 	t.Parallel()
-	ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{Level: log.Debug}))
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
 	ctx, cancel := context.WithCancel(ctx)
 	t.Cleanup(cancel)
 

backend/schema/module.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"database/sql/driver"
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 
@@ -175,6 +176,15 @@ func (m *Module) ToProto() proto.Message {
 	}
 }
 
+// ModuleFromProtoFile loads a module from the given proto-encoded file.
+func ModuleFromProtoFile(filename string) (*Module, error) {
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return ModuleFromBytes(data)
+}
+
 // ModuleFromProto converts a protobuf Module to a Module and validates it.
 func ModuleFromProto(s *schemapb.Module) (*Module, error) {
 	module := &Module{

buildengine/build.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/moduleconfig"
+	"github.com/TBD54566975/ftl/internal/log"
 )
 
 // A Module is a ModuleConfig with its dependencies populated.
@@ -17,22 +19,25 @@ type Module struct {
 //
 // A [Module] includes the module configuration as well as its dependencies
 // extracted from source code.
-func LoadModule(dir string) (Module, error) {
+func LoadModule(ctx context.Context, dir string) (Module, error) {
 	config, err := moduleconfig.LoadModuleConfig(dir)
 	if err != nil {
 		return Module{}, err
 	}
-	return UpdateDependencies(config)
+	return UpdateDependencies(ctx, config)
 }
 
 // Build a module in the given directory given the schema and module config.
-func Build(ctx context.Context /*schema *schema.Schema, */, module Module) error {
+func Build(ctx context.Context, sch *schema.Schema, module Module) error {
+	logger := log.FromContext(ctx).Scope(module.Module)
+	ctx = log.ContextWithLogger(ctx, logger)
+	logger.Infof("Building module")
 	switch module.Language {
 	case "go":
-		return buildGo(ctx, module)
+		return buildGo(ctx, sch, module)
 
 	case "kotlin":
-		return buildKotlin(ctx, module)
+		return buildKotlin(ctx, sch, module)
 
 	default:
 		return fmt.Errorf("unknown language %q", module.Language)

buildengine/build_go.go

@@ -4,25 +4,11 @@ import (
 	"context"
 	"fmt"
 
-	"connectrpc.com/connect"
-
-	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
-	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/go-runtime/compile"
-	"github.com/TBD54566975/ftl/internal/rpc"
 )
 
-func buildGo(ctx context.Context, module Module) error {
-	client := rpc.ClientFromContext[ftlv1connect.ControllerServiceClient](ctx)
-	resp, err := client.GetSchema(ctx, connect.NewRequest(&ftlv1.GetSchemaRequest{}))
-	if err != nil {
-		return err
-	}
-	sch, err := schema.FromProto(resp.Msg.Schema)
-	if err != nil {
-		return fmt.Errorf("failed to convert schema from proto: %w", err)
-	}
+func buildGo(ctx context.Context, sch *schema.Schema, module Module) error {
 	if err := compile.Build(ctx, module.Dir, sch); err != nil {
 		return fmt.Errorf("failed to build module: %w", err)
 	}

buildengine/build_kotlin.go

@@ -9,11 +9,12 @@ import (
 	"github.com/beevik/etree"
 
 	"github.com/TBD54566975/ftl"
+	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/internal/exec"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
-func buildKotlin(ctx context.Context, module Module) error {
+func buildKotlin(ctx context.Context, _ *schema.Schema, module Module) error {
 	logger := log.FromContext(ctx)
 
 	if err := SetPOMProperties(ctx, filepath.Join(module.Dir, "..")); err != nil {

buildengine/deps.go

@@ -2,6 +2,7 @@ package buildengine
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"go/parser"
 	"go/token"
@@ -17,13 +18,14 @@ import (
 	"golang.org/x/exp/maps"
 
 	"github.com/TBD54566975/ftl/common/moduleconfig"
+	"github.com/TBD54566975/ftl/internal/log"
 )
 
 // UpdateAllDependencies calls UpdateDependencies on each module in the list.
-func UpdateAllDependencies(modules []moduleconfig.ModuleConfig) ([]Module, error) {
+func UpdateAllDependencies(ctx context.Context, modules ...moduleconfig.ModuleConfig) ([]Module, error) {
 	out := []Module{}
 	for _, module := range modules {
-		updated, err := UpdateDependencies(module)
+		updated, err := UpdateDependencies(ctx, module)
 		if err != nil {
 			return nil, err
 		}
@@ -34,7 +36,9 @@ func UpdateAllDependencies(modules []moduleconfig.ModuleConfig) ([]Module, error
 
 // UpdateDependencies finds the dependencies for an FTL module and returns a
 // Module with those dependencies populated.
-func UpdateDependencies(config moduleconfig.ModuleConfig) (Module, error) {
+func UpdateDependencies(ctx context.Context, config moduleconfig.ModuleConfig) (Module, error) {
+	logger := log.FromContext(ctx)
+	logger.Debugf("Extracting dependencies for module %s", config.Module)
 	dependencies, err := extractDependencies(config)
 	if err != nil {
 		return Module{}, err
@@ -63,6 +67,9 @@ func extractGoFTLImports(self, dir string) ([]string, error) {
 		if !d.IsDir() {
 			return nil
 		}
+		if strings.HasPrefix(d.Name(), "_") || d.Name() == "testdata" {
+			return ErrSkip
+		}
 		pkgs, err := parser.ParseDir(fset, path, nil, parser.ImportsOnly)
 		if pkgs == nil {
 			return err

buildengine/discover.go

@@ -1,6 +1,7 @@
 package buildengine
 
 import (
+	"context"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -12,7 +13,7 @@ import (
 // DiscoverModules recursively loads all modules under the given directories.
 //
 // If no directories are provided, the current working directory is used.
-func DiscoverModules(dirs ...string) ([]moduleconfig.ModuleConfig, error) {
+func DiscoverModules(ctx context.Context, dirs ...string) ([]moduleconfig.ModuleConfig, error) {
 	if len(dirs) == 0 {
 		cwd, err := os.Getwd()
 		if err != nil {

buildengine/discover_test.go

@@ -1,15 +1,18 @@
 package buildengine
 
 import (
+	"context"
 	"testing"
 
 	"github.com/alecthomas/assert/v2"
 
 	"github.com/TBD54566975/ftl/common/moduleconfig"
+	"github.com/TBD54566975/ftl/internal/log"
 )
 
 func TestDiscoverModules(t *testing.T) {
-	modules, err := DiscoverModules("testdata/modules")
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
+	modules, err := DiscoverModules(ctx, "testdata/modules")
 	assert.NoError(t, err)
 	expected := []moduleconfig.ModuleConfig{
 		{