chore: refactor go schema extraction

initial step towards #1518

- integrates with (forked) go/analysis to run static analysis tasks for schema extraction
- implements first extraction task, type aliases
- refactors existing build stack to use the new extractor, combining results with legacy so we can incrementally migrate
- removes typealias extraction logic from legacy code

backend/schema/errors.go

@@ -125,3 +125,12 @@ func SortErrorsByPosition(merr []*Error) {
 			(ipp.Line == jpp.Line && ipp.Column == jpp.Column && merr[i].EndColumn == merr[j].EndColumn && merr[i].Msg < merr[j].Msg)
 	})
 }
+
+func ContainsTerminalError(errs []*Error) bool {
+	for _, e := range errs {
+		if e.Level == ERROR {
+			return true
+		}
+	}
+	return false
+}

backend/schema/module.go

@@ -117,6 +117,54 @@ func (m *Module) String() string {
 	return w.String()
 }
 
+// Merge the contents of another module schema into this one. Does not handle comments. Errors if the module names
+// do not match.
+func (m *Module) Merge(other *Module) error {
+	if other == nil {
+		return nil
+	}
+	if m.Name != other.Name {
+		return fmt.Errorf("cannot merge modules %s and %s", m.Name, other.Name)
+	}
+	m.AddDecls(other.Decls)
+	return nil
+}
+
+// AddDecls appends decls to the module.
+//
+// Decls are only added if they are not already present in the module or if they change the visibility of an existing
+// Decl.
+func (m *Module) AddDecls(decls []Decl) {
+	// Decls are namespaced by their type.
+	typeQualifiedName := func(d Decl) string {
+		return reflect.TypeOf(d).Name() + "." + d.GetName()
+	}
+
+	existingDecls := map[string]Decl{}
+	for _, d := range m.Decls {
+		existingDecls[typeQualifiedName(d)] = d
+	}
+	for _, newDecl := range decls {
+		tqName := typeQualifiedName(newDecl)
+		if existingDecl, ok := existingDecls[tqName]; ok {
+			if newDecl.IsExported() && !existingDecl.IsExported() {
+				existingDecls[tqName] = newDecl
+			}
+			continue
+		}
+
+		existingDecls[tqName] = newDecl
+	}
+	m.Decls = maps.Values(existingDecls)
+}
+
+// AddDecl adds a single decl to the module.
+//
+// It is only added if not already present or if it changes the visibility of the existing Decl.
+func (m *Module) AddDecl(decl Decl) {
+	m.AddDecls([]Decl{decl})
+}
+
 // AddData and return its index.
 //
 // If data is already in the module, the existing index is returned.

go-runtime/compile/build.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"maps"
-	"net"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,6 +14,8 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/TBD54566975/ftl/go-runtime/schema/analyzers"
+	"github.com/alecthomas/types/optional"
 	sets "github.com/deckarep/golang-set/v2"
 	gomaps "golang.org/x/exp/maps"
 	"golang.org/x/mod/modfile"
@@ -27,6 +28,7 @@ import (
 	"github.com/TBD54566975/ftl"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/common/moduleconfig"
+	extract "github.com/TBD54566975/ftl/go-runtime/schema"
 	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/exec"
 	"github.com/TBD54566975/ftl/internal/log"
@@ -139,39 +141,35 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans
 
 	buildDir := buildDir(moduleDir)
 	logger.Debugf("Extracting schema")
-	parseResult, err := ExtractModuleSchema(moduleDir, sch)
+	maybeResult, err := ExtractModuleSchema(config.Dir, sch)
 	if err != nil {
-		return fmt.Errorf("failed to extract module schema: %w", err)
+		return err
 	}
-	pr, ok := parseResult.Get()
+	result, ok := maybeResult.Get()
 	if !ok {
-		return fmt.Errorf("failed to extract module schema")
+		return fmt.Errorf("could not extract schema from module %q", moduleDir)
 	}
 
-	main := pr.Module
-	if schemaErrs := pr.Errors; len(schemaErrs) > 0 {
-		if err := writeSchemaErrors(config, schemaErrs); err != nil {
-			return fmt.Errorf("failed to write errors: %w", err)
-		}
-		return nil
+	if err = writeSchemaErrors(config, result.Errors); err != nil {
+		return fmt.Errorf("failed to write schema errors: %w", err)
 	}
-	schemaBytes, err := proto.Marshal(main.ToProto())
-	if err != nil {
-		return fmt.Errorf("failed to marshal schema: %w", err)
+	if schema.ContainsTerminalError(result.Errors) {
+		// Only bail if schema errors contain elements at level ERROR.
+		// If errors are only at levels below ERROR (e.g. INFO, WARN), the schema can still be used.
+		return nil
 	}
-	err = os.WriteFile(filepath.Join(buildDir, "schema.pb"), schemaBytes, 0600)
-	if err != nil {
+	if err = writeSchema(config, result.Module); err != nil {
 		return fmt.Errorf("failed to write schema: %w", err)
 	}
 
 	logger.Debugf("Generating main module")
-	goVerbs := make([]goVerb, 0, len(main.Decls))
-	for _, decl := range main.Decls {
+	goVerbs := make([]goVerb, 0, len(result.Module.Decls))
+	for _, decl := range result.Module.Decls {
 		verb, ok := decl.(*schema.Verb)
 		if !ok {
 			continue
 		}
-		nativeName, ok := pr.NativeNames[verb]
+		nativeName, ok := result.NativeNames[verb]
 		if !ok {
 			return fmt.Errorf("missing native name for verb %s", verb.Name)
 		}
@@ -188,10 +186,10 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans
 	if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{
 		GoVersion:    goModVersion,
 		FTLVersion:   ftlVersion,
-		Name:         main.Name,
+		Name:         result.Module.Name,
 		Verbs:        goVerbs,
 		Replacements: replacements,
-		SumTypes:     getSumTypes(main, sch, pr.NativeNames),
+		SumTypes:     getSumTypes(result.Module, sch, result.NativeNames),
 	}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil {
 		return err
 	}
@@ -226,6 +224,36 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans
 	return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx)
 }
 
+// ExtractModuleSchema statically parses Go FTL module source into a schema.Module.
+func ExtractModuleSchema(dir string, sch *schema.Schema) (optional.Option[analyzers.ExtractResult], error) {
+	maybeResult, err := extract.Extract(dir)
+	if err != nil {
+		return optional.None[analyzers.ExtractResult](), err
+	}
+	result, ok := maybeResult.Get()
+	if !ok {
+		return optional.None[analyzers.ExtractResult](), fmt.Errorf("could not extract schema from module %q", dir)
+	}
+
+	// TODO: Remove legacy schema extraction when possible
+	if err = legacyExtractModuleSchema(dir, sch, &result); err != nil {
+		return optional.None[analyzers.ExtractResult](), err
+	}
+
+	schema.SortErrorsByPosition(result.Errors)
+	if !schema.ContainsTerminalError(result.Errors) {
+		err = schema.ValidateModule(result.Module)
+		if err != nil {
+			return optional.None[analyzers.ExtractResult](), err
+		}
+	}
+	if len(result.Errors) == 0 {
+		result.Errors = nil
+	}
+
+	return optional.Some(result), nil
+}
+
 func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error {
 	goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod"))
 	if err != nil {
@@ -257,11 +285,6 @@ func generateExternalModules(context ExternalModuleContext) error {
 	return internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs))
 }
 
-func online() bool {
-	_, err := net.LookupHost("proxy.golang.org")
-	return err == nil
-}
-
 var scaffoldFuncs = scaffolder.FuncMap{
 	"comment": schema.EncodeComments,
 	"type":    genType,
@@ -525,6 +548,14 @@ func shouldUpdateVersion(goModfile *modfile.File) bool {
 	return true
 }
 
+func writeSchema(config moduleconfig.ModuleConfig, module *schema.Module) error {
+	schemaBytes, err := proto.Marshal(module.ToProto())
+	if err != nil {
+		return fmt.Errorf("failed to marshal schema: %w", err)
+	}
+	return os.WriteFile(filepath.Join(config.AbsDeployDir(), "schema.pb"), schemaBytes, 0600)
+}
+
 func writeSchemaErrors(config moduleconfig.ModuleConfig, errors []*schema.Error) error {
 	el := schema.ErrorList{
 		Errors: errors,

go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl

@@ -21,11 +21,11 @@ import (
 var _ = context.Background
 
 {{- range .Decls }}
+{{- if .IsExported}}
 {{if .Comments}}
 {{.Comments|comment -}}
 //
 {{- end}}
-{{- if .IsExported}}
 {{- if is "Topic" .}}
 var {{.Name|title}} = ftl.Topic[{{type $ .Event}}]("{{.Name}}")
 {{- else if and (is "Enum" .) .IsValueEnum}}

go-runtime/compile/parser.go

@@ -1,5 +1,8 @@
 package compile
 
+// TODO: This file is now duplicated in go-runtime/schema/analyzers/parser.go. It should be removed
+// from here once the schema extraction refactoring is complete.
+
 import (
 	"errors"
 	"fmt"