feat: produce errors.pb during go schema extraction

This approach enables arbitrary runtimes to surface schema errors via proto. Go will use this approach for consistency across runtimes.

fixes #1207

backend/schema/errors.go

@@ -11,7 +11,14 @@ type Error struct {
 	Msg       string   `json:"msg" protobuf:"1"`
 	Pos       Position `json:"pos" protobuf:"2"`
 	EndColumn int      `json:"endCol" protobuf:"3"`
-	Err       error    `protobuf:"-"` // Wrapped error, if any
+}
+
+func (e *Error) ToProto() *schemapb.Error {
+	return &schemapb.Error{
+		Msg:       e.Msg,
+		Pos:       posToProto(e.Pos),
+		EndColumn: int64(e.EndColumn),
+	}
 }
 
 func errorFromProto(e *schemapb.Error) *Error {
@@ -22,6 +29,15 @@ func errorFromProto(e *schemapb.Error) *Error {
 	}
 }
 
+func errorsToProto(errs []*Error) []*schemapb.Error {
+	var out []*schemapb.Error
+	for _, s := range errs {
+		pb := s.ToProto()
+		out = append(out, pb)
+	}
+	return out
+}
+
 func errorsFromProto(errs []*schemapb.Error) []*Error {
 	var out []*Error
 	for _, pb := range errs {
@@ -35,6 +51,12 @@ type ErrorList struct {
 	Errors []*Error `json:"errors" protobuf:"1"`
 }
 
+func (e *ErrorList) ToProto() *schemapb.ErrorList {
+	return &schemapb.ErrorList{
+		Errors: errorsToProto(e.Errors),
+	}
+}
+
 // ErrorListFromProto converts a protobuf ErrorList to an ErrorList.
 func ErrorListFromProto(e *schemapb.ErrorList) *ErrorList {
 	return &ErrorList{
@@ -43,7 +65,7 @@ func ErrorListFromProto(e *schemapb.ErrorList) *ErrorList {
 }
 
 func (e Error) Error() string { return fmt.Sprintf("%s-%d: %s", e.Pos, e.EndColumn, e.Msg) }
-func (e Error) Unwrap() error { return e.Err }
+func (e Error) Unwrap() error { return nil }
 
 func Errorf(pos Position, endColumn int, format string, args ...any) Error {
 	return Error{Msg: fmt.Sprintf(format, args...), Pos: pos, EndColumn: endColumn}
@@ -67,5 +89,5 @@ func Wrapf(pos Position, endColumn int, err error, format string, args ...any) E
 		newEndColumn = endColumn
 		args = append(args, err)
 	}
-	return Error{Msg: fmt.Sprintf(format, args...), Pos: newPos, EndColumn: newEndColumn, Err: err}
+	return Error{Msg: fmt.Sprintf(format, args...), Pos: newPos, EndColumn: newEndColumn}
 }

backend/schema/validate.go

@@ -9,13 +9,11 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/alecthomas/participle/v2"
-	"github.com/alecthomas/types/optional"
-	"golang.org/x/exp/maps"
-
 	"github.com/TBD54566975/ftl/internal/cron"
 	"github.com/TBD54566975/ftl/internal/errors"
 	dc "github.com/TBD54566975/ftl/internal/reflect"
+	"github.com/alecthomas/participle/v2"
+	"github.com/alecthomas/types/optional"
 )
 
 var (
@@ -353,14 +351,7 @@ func cleanErrors(merr []error) []error {
 	if len(merr) == 0 {
 		return nil
 	}
-	// Deduplicate.
-	set := map[string]error{}
-	for _, err := range merr {
-		for _, subErr := range errors.UnwrapAll(err) {
-			set[strings.TrimSpace(subErr.Error())] = subErr
-		}
-	}
-	merr = maps.Values(set)
+	merr = errors.DeduplicateErrors(merr)
 	// Sort by position.
 	sort.Slice(merr, func(i, j int) bool {
 		var ipe, jpe participle.Error

buildengine/build.go

@@ -3,6 +3,10 @@ package buildengine
 import (
 	"context"
 	"fmt"
+	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema"
+	"github.com/TBD54566975/ftl/internal/errors"
+	"google.golang.org/protobuf/proto"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -29,14 +33,30 @@ func buildModule(ctx context.Context, sch *schema.Schema, module Module) error {
 	ctx = log.ContextWithLogger(ctx, logger)
 
 	logger.Infof("Building module")
+	var err error
 	switch module.Language {
 	case "go":
-		return buildGoModule(ctx, sch, module)
+		err = buildGoModule(ctx, sch, module)
 	case "kotlin":
-		return buildKotlinModule(ctx, sch, module)
+		err = buildKotlinModule(ctx, sch, module)
 	default:
 		return fmt.Errorf("unknown language %q", module.Language)
 	}
+
+	if err != nil {
+		// read runtime-specific build errors from the build directory
+		errorList, err := loadProtoErrors(module.AbsDeployDir())
+		if err != nil {
+			return fmt.Errorf("failed to read build errors for module: %w", err)
+		}
+		errs := make([]error, 0, len(errorList.Errors))
+		for _, e := range errorList.Errors {
+			errs = append(errs, *e)
+		}
+		return errors.Join(errs...)
+	}
+
+	return nil
 }
 
 func buildExternalLibrary(ctx context.Context, sch *schema.Schema, lib ExternalLibrary) error {
@@ -64,3 +84,21 @@ func buildExternalLibrary(ctx context.Context, sch *schema.Schema, lib ExternalL
 	logger.Infof("Generated stubs [%s] for %v", strings.Join(imported, ", "), lib)
 	return nil
 }
+
+func loadProtoErrors(buildDir string) (*schema.ErrorList, error) {
+	f := filepath.Join(buildDir, "errors.pb")
+	if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) {
+		return &schema.ErrorList{Errors: make([]*schema.Error, 0)}, nil
+	}
+
+	content, err := os.ReadFile(f)
+	if err != nil {
+		return nil, err
+	}
+	errorspb := &schemapb.ErrorList{}
+	err = proto.Unmarshal(content, errorspb)
+	if err != nil {
+		return nil, err
+	}
+	return schema.ErrorListFromProto(errorspb), nil
+}

buildengine/build_go_test.go

@@ -1,14 +1,9 @@
 package buildengine
 
 import (
-	"context"
-	"os"
 	"testing"
 
-	"github.com/alecthomas/assert/v2"
-
 	"github.com/TBD54566975/ftl/backend/schema"
-	"github.com/TBD54566975/ftl/internal/log"
 )
 
 func TestGenerateGoModule(t *testing.T) {
@@ -183,17 +178,12 @@ func TestExternalType(t *testing.T) {
 	if testing.Short() {
 		t.SkipNow()
 	}
-	moduleDir := "testdata/projects/external"
-	buildDir := "_ftl"
-
-	ctx := log.ContextWithLogger(context.Background(), log.Configure(os.Stderr, log.Config{}))
-	module, err := LoadModule(moduleDir)
-	assert.NoError(t, err)
-
-	sch := &schema.Schema{}
-	err = Build(ctx, sch, module)
-	assert.Contains(t, err.Error(), "field Month: unsupported external type time.Month")
-
-	err = os.RemoveAll(buildDir)
-	assert.NoError(t, err, "Error removing build directory")
+	bctx := buildContext{
+		moduleDir: "testdata/projects/external",
+		buildDir:  "_ftl",
+		sch:       &schema.Schema{},
+	}
+	testBuild(t, bctx, true, []assertion{
+		assertBuildProtoErrors("field Month: unsupported external type time.Month"),
+	})
 }

buildengine/build_kotlin.go

@@ -2,27 +2,23 @@ package buildengine
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 
-	"github.com/TBD54566975/scaffolder"
-	"github.com/beevik/etree"
-	sets "github.com/deckarep/golang-set/v2"
-	"golang.org/x/exp/maps"
-	"google.golang.org/protobuf/proto"
-
 	"github.com/TBD54566975/ftl"
-	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema"
 	"github.com/TBD54566975/ftl/backend/schema"
 	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/exec"
 	"github.com/TBD54566975/ftl/internal/log"
 	kotlinruntime "github.com/TBD54566975/ftl/kotlin-runtime"
+	"github.com/TBD54566975/scaffolder"
+	"github.com/beevik/etree"
+	sets "github.com/deckarep/golang-set/v2"
+	"golang.org/x/exp/maps"
 )
 
 type externalModuleContext struct {
@@ -62,16 +58,7 @@ func buildKotlinModule(ctx context.Context, sch *schema.Schema, module Module) e
 	logger.Debugf("Using build command '%s'", module.Build)
 	err := exec.Command(ctx, log.Debug, module.Dir, "bash", "-c", module.Build).RunBuffered(ctx)
 	if err != nil {
-		// read runtime-specific build errors from the build directory
-		errorList, err := loadProtoErrors(module.AbsDeployDir())
-		if err != nil {
-			return fmt.Errorf("failed to read build errors for module: %w", err)
-		}
-		errs := make([]error, 0, len(errorList.Errors))
-		for _, e := range errorList.Errors {
-			errs = append(errs, *e)
-		}
-		return errors.Join(errs...)
+		return fmt.Errorf("failed to build module %q: %w", module.Module, err)
 	}
 
 	return nil
@@ -261,21 +248,3 @@ func genType(module *schema.Module, t schema.Type) string {
 	}
 	panic(fmt.Sprintf("unsupported type %T", t))
 }
-
-func loadProtoErrors(buildDir string) (*schema.ErrorList, error) {
-	f := filepath.Join(buildDir, "errors.pb")
-	if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) {
-		return &schema.ErrorList{Errors: make([]*schema.Error, 0)}, nil
-	}
-
-	content, err := os.ReadFile(f)
-	if err != nil {
-		return nil, err
-	}
-	errorspb := &schemapb.ErrorList{}
-	err = proto.Unmarshal(content, errorspb)
-	if err != nil {
-		return nil, err
-	}
-	return schema.ErrorListFromProto(errorspb), nil
-}

go-runtime/compile/build.go

@@ -3,6 +3,7 @@ package compile
 import (
 	"context"
 	"fmt"
+	"github.com/TBD54566975/ftl/internal/errors"
 	"maps"
 	"net"
 	"os"
@@ -98,16 +99,39 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema) error {
 		return fmt.Errorf("failed to generate external modules: %w", err)
 	}
 
+	buildDir := buildDir(moduleDir)
 	logger.Debugf("Extracting schema")
 	nativeNames, main, err := ExtractModuleSchema(moduleDir)
 	if err != nil {
-		return fmt.Errorf("failed to extract module schema: %w", err)
+		var schemaErrs []*schema.Error
+		var otherErrs []error
+		for _, e := range errors.DeduplicateErrors(errors.UnwrapAll(err)) {
+			var ce schema.Error
+			if errors.As(e, &ce) {
+				schemaErrs = append(schemaErrs, &ce)
+			} else {
+				otherErrs = append(otherErrs, e)
+			}
+		}
+		el := schema.ErrorList{
+			Errors: schemaErrs,
+		}
+		elBytes, err := proto.Marshal(el.ToProto())
+		if err != nil {
+			return fmt.Errorf("failed to marshal errors: %w", err)
+		}
+
+		err = os.WriteFile(filepath.Join(buildDir, "errors.pb"), elBytes, 0600)
+		if err != nil {
+			return fmt.Errorf("failed to write errors: %w", err)
+		}
+
+		return fmt.Errorf("failed to extract module schema: %w", errors.Join(otherErrs...))
 	}
 	schemaBytes, err := proto.Marshal(main.ToProto())
 	if err != nil {
 		return fmt.Errorf("failed to marshal schema: %w", err)
 	}
-	buildDir := buildDir(moduleDir)
 	err = os.WriteFile(filepath.Join(buildDir, "schema.pb"), schemaBytes, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to write schema: %w", err)

internal/errors/errors.go

@@ -1,6 +1,10 @@
 package errors
 
-import "errors"
+import (
+	"errors"
+	"golang.org/x/exp/maps"
+	"strings"
+)
 
 // UnwrapAll recursively unwraps all errors in err, including all intermediate errors.
 //
@@ -42,3 +46,14 @@ func As(err error, target interface{}) bool { return errors.As(err, target) }
 func Is(err, target error) bool { return errors.Is(err, target) }
 
 func Unwrap(err error) error { return errors.Unwrap(err) }
+
+// DeduplicateErrors de-duplicates equivalent errors.
+func DeduplicateErrors(merr []error) []error {
+	set := map[string]error{}
+	for _, err := range merr {
+		for _, subErr := range UnwrapAll(err) {
+			set[strings.TrimSpace(subErr.Error())] = subErr
+		}
+	}
+	return maps.Values(set)
+}

lsp/lsp.go

@@ -77,18 +77,16 @@ func (s *Server) post(err error) {
 	errByFilename := make(map[string]errSet)
 
 	// Deduplicate and associate by filename.
-	errs := ftlErrors.UnwrapAll(err)
-	for _, err := range errs {
-		if ftlErrors.Innermost(err) {
-			var ce schema.Error
-			if errors.As(err, &ce) {
-				filename := ce.Pos.Filename
-				if _, exists := errByFilename[filename]; !exists {
-					errByFilename[filename] = make(errSet)
-				}
-				errByFilename[filename][strings.TrimSpace(ce.Error())] = ce
+	for _, err := range ftlErrors.DeduplicateErrors(ftlErrors.UnwrapAll(err)) {
+		var ce schema.Error
+		if errors.As(err, &ce) {
+			filename := ce.Pos.Filename
+			if _, exists := errByFilename[filename]; !exists {
+				errByFilename[filename] = make(errSet)
 			}
+			errByFilename[filename][strings.TrimSpace(ce.Error())] = ce
 		}
+
 	}
 
 	go publishErrors(errByFilename, s)