diff --git a/backend/schema/errors.go b/backend/schema/errors.go index 851b8db4c6..8acbddfd2c 100644 --- a/backend/schema/errors.go +++ b/backend/schema/errors.go @@ -125,3 +125,12 @@ func SortErrorsByPosition(merr []*Error) { (ipp.Line == jpp.Line && ipp.Column == jpp.Column && merr[i].EndColumn == merr[j].EndColumn && merr[i].Msg < merr[j].Msg) }) } + +func ContainsTerminalError(errs []*Error) bool { + for _, e := range errs { + if e.Level == ERROR { + return true + } + } + return false +} diff --git a/backend/schema/module.go b/backend/schema/module.go index 7693b99893..3db070c0ea 100644 --- a/backend/schema/module.go +++ b/backend/schema/module.go @@ -117,6 +117,54 @@ func (m *Module) String() string { return w.String() } +// Merge the contents of another module schema into this one. Does not handle comments. Errors if the module names +// do not match. +func (m *Module) Merge(other *Module) error { + if other == nil { + return nil + } + if m.Name != other.Name { + return fmt.Errorf("cannot merge modules %s and %s", m.Name, other.Name) + } + m.AddDecls(other.Decls) + return nil +} + +// AddDecls appends decls to the module. +// +// Decls are only added if they are not already present in the module or if they change the visibility of an existing +// Decl. +func (m *Module) AddDecls(decls []Decl) { + // Decls are namespaced by their type. + typeQualifiedName := func(d Decl) string { + return reflect.TypeOf(d).Name() + "." + d.GetName() + } + + existingDecls := map[string]Decl{} + for _, d := range m.Decls { + existingDecls[typeQualifiedName(d)] = d + } + for _, newDecl := range decls { + tqName := typeQualifiedName(newDecl) + if existingDecl, ok := existingDecls[tqName]; ok { + if newDecl.IsExported() && !existingDecl.IsExported() { + existingDecls[tqName] = newDecl + } + continue + } + + existingDecls[tqName] = newDecl + } + m.Decls = maps.Values(existingDecls) +} + +// AddDecl adds a single decl to the module. +// +// It is only added if not already present or if it changes the visibility of the existing Decl. +func (m *Module) AddDecl(decl Decl) { + m.AddDecls([]Decl{decl}) +} + // AddData and return its index. // // If data is already in the module, the existing index is returned. diff --git a/go-runtime/compile/build.go b/go-runtime/compile/build.go index 5a43b3df0a..35c374c982 100644 --- a/go-runtime/compile/build.go +++ b/go-runtime/compile/build.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "maps" - "net" "os" "path" "path/filepath" @@ -15,6 +14,8 @@ import ( "strings" "unicode" + "github.com/TBD54566975/ftl/go-runtime/schema/analyzers" + "github.com/alecthomas/types/optional" sets "github.com/deckarep/golang-set/v2" gomaps "golang.org/x/exp/maps" "golang.org/x/mod/modfile" @@ -27,6 +28,7 @@ import ( "github.com/TBD54566975/ftl" "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/common/moduleconfig" + extract "github.com/TBD54566975/ftl/go-runtime/schema" "github.com/TBD54566975/ftl/internal" "github.com/TBD54566975/ftl/internal/exec" "github.com/TBD54566975/ftl/internal/log" @@ -139,39 +141,35 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans buildDir := buildDir(moduleDir) logger.Debugf("Extracting schema") - parseResult, err := ExtractModuleSchema(moduleDir, sch) + maybeResult, err := ExtractModuleSchema(config.Dir, sch) if err != nil { - return fmt.Errorf("failed to extract module schema: %w", err) + return err } - pr, ok := parseResult.Get() + result, ok := maybeResult.Get() if !ok { - return fmt.Errorf("failed to extract module schema") + return fmt.Errorf("could not extract schema from module %q", moduleDir) } - main := pr.Module - if schemaErrs := pr.Errors; len(schemaErrs) > 0 { - if err := writeSchemaErrors(config, schemaErrs); err != nil { - return fmt.Errorf("failed to write errors: %w", err) - } - return nil + if err = writeSchemaErrors(config, result.Errors); err != nil { + return fmt.Errorf("failed to write schema errors: %w", err) } - schemaBytes, err := proto.Marshal(main.ToProto()) - if err != nil { - return fmt.Errorf("failed to marshal schema: %w", err) + if schema.ContainsTerminalError(result.Errors) { + // Only bail if schema errors contain elements at level ERROR. + // If errors are only at levels below ERROR (e.g. INFO, WARN), the schema can still be used. + return nil } - err = os.WriteFile(filepath.Join(buildDir, "schema.pb"), schemaBytes, 0600) - if err != nil { + if err = writeSchema(config, result.Module); err != nil { return fmt.Errorf("failed to write schema: %w", err) } logger.Debugf("Generating main module") - goVerbs := make([]goVerb, 0, len(main.Decls)) - for _, decl := range main.Decls { + goVerbs := make([]goVerb, 0, len(result.Module.Decls)) + for _, decl := range result.Module.Decls { verb, ok := decl.(*schema.Verb) if !ok { continue } - nativeName, ok := pr.NativeNames[verb] + nativeName, ok := result.NativeNames[verb] if !ok { return fmt.Errorf("missing native name for verb %s", verb.Name) } @@ -188,10 +186,10 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mainModuleContext{ GoVersion: goModVersion, FTLVersion: ftlVersion, - Name: main.Name, + Name: result.Module.Name, Verbs: goVerbs, Replacements: replacements, - SumTypes: getSumTypes(main, sch, pr.NativeNames), + SumTypes: getSumTypes(result.Module, sch, result.NativeNames), }, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil { return err } @@ -226,6 +224,36 @@ func Build(ctx context.Context, moduleDir string, sch *schema.Schema, filesTrans return exec.Command(ctx, log.Debug, mainDir, "go", "build", "-o", "../../main", ".").RunBuffered(ctx) } +// ExtractModuleSchema statically parses Go FTL module source into a schema.Module. +func ExtractModuleSchema(dir string, sch *schema.Schema) (optional.Option[analyzers.ExtractResult], error) { + maybeResult, err := extract.Extract(dir) + if err != nil { + return optional.None[analyzers.ExtractResult](), err + } + result, ok := maybeResult.Get() + if !ok { + return optional.None[analyzers.ExtractResult](), fmt.Errorf("could not extract schema from module %q", dir) + } + + // TODO: Remove legacy schema extraction when possible + if err = legacyExtractModuleSchema(dir, sch, &result); err != nil { + return optional.None[analyzers.ExtractResult](), err + } + + schema.SortErrorsByPosition(result.Errors) + if !schema.ContainsTerminalError(result.Errors) { + err = schema.ValidateModule(result.Module) + if err != nil { + return optional.None[analyzers.ExtractResult](), err + } + } + if len(result.Errors) == 0 { + result.Errors = nil + } + + return optional.Some(result), nil +} + func GenerateStubsForExternalLibrary(ctx context.Context, dir string, schema *schema.Schema) error { goModFile, replacements, err := goModFileWithReplacements(filepath.Join(dir, "go.mod")) if err != nil { @@ -257,11 +285,6 @@ func generateExternalModules(context ExternalModuleContext) error { return internal.ScaffoldZip(externalModuleTemplateFiles(), context.ModuleDir, context, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)) } -func online() bool { - _, err := net.LookupHost("proxy.golang.org") - return err == nil -} - var scaffoldFuncs = scaffolder.FuncMap{ "comment": schema.EncodeComments, "type": genType, @@ -525,6 +548,14 @@ func shouldUpdateVersion(goModfile *modfile.File) bool { return true } +func writeSchema(config moduleconfig.ModuleConfig, module *schema.Module) error { + schemaBytes, err := proto.Marshal(module.ToProto()) + if err != nil { + return fmt.Errorf("failed to marshal schema: %w", err) + } + return os.WriteFile(filepath.Join(config.AbsDeployDir(), "schema.pb"), schemaBytes, 0600) +} + func writeSchemaErrors(config moduleconfig.ModuleConfig, errors []*schema.Error) error { el := schema.ErrorList{ Errors: errors, diff --git a/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl b/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl index f55619c0e5..06ce2d82b0 100644 --- a/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl +++ b/go-runtime/compile/external-module-template/_ftl/go/modules/{{ range .NonMainModules }}{{ push .Name . }}{{ end }}/external_module.go.tmpl @@ -21,11 +21,11 @@ import ( var _ = context.Background {{- range .Decls }} +{{- if .IsExported}} {{if .Comments}} {{.Comments|comment -}} // {{- end}} -{{- if .IsExported}} {{- if is "Topic" .}} var {{.Name|title}} = ftl.Topic[{{type $ .Event}}]("{{.Name}}") {{- else if and (is "Enum" .) .IsValueEnum}} diff --git a/go-runtime/compile/parser.go b/go-runtime/compile/parser.go index 2ba69a2057..292195ff53 100644 --- a/go-runtime/compile/parser.go +++ b/go-runtime/compile/parser.go @@ -1,5 +1,8 @@ package compile +// TODO: This file is now duplicated in go-runtime/schema/analyzers/parser.go. It should be removed +// from here once the schema extraction refactoring is complete. + import ( "errors" "fmt" diff --git a/go-runtime/compile/schema.go b/go-runtime/compile/schema.go index b71a2e67d7..3522896fa5 100644 --- a/go-runtime/compile/schema.go +++ b/go-runtime/compile/schema.go @@ -14,6 +14,7 @@ import ( "unicode" "unicode/utf8" + "github.com/TBD54566975/ftl/go-runtime/schema/analyzers" "github.com/alecthomas/types/optional" "golang.org/x/exp/maps" "golang.org/x/tools/go/ast/astutil" @@ -21,7 +22,6 @@ import ( "github.com/TBD54566975/ftl/backend/schema" "github.com/TBD54566975/ftl/backend/schema/strcase" - "github.com/TBD54566975/ftl/internal/errors" "github.com/TBD54566975/ftl/internal/goast" ) @@ -106,44 +106,28 @@ type ParseResult struct { Errors []*schema.Error } -// ExtractModuleSchema statically parses Go FTL module source into a schema.Module. -func ExtractModuleSchema(dir string, sch *schema.Schema) (optional.Option[ParseResult], error) { +func legacyExtractModuleSchema(dir string, sch *schema.Schema, out *analyzers.ExtractResult) error { pkgs, err := packages.Load(&packages.Config{ Dir: dir, Fset: fset, Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports, }, "./...") if err != nil { - return optional.None[ParseResult](), err + return err } if len(pkgs) == 0 { - return optional.None[ParseResult](), fmt.Errorf("no packages found in %q, does \"go mod tidy\" need to be run?", dir) + return fmt.Errorf("no packages found in %q, does \"go mod tidy\" need to be run?", dir) } - // Find module name - module := &schema.Module{} - merr := []error{} - schemaErrs := []*schema.Error{} - nativeNames := NativeNames{} for _, pkg := range pkgs { - moduleName, ok := ftlModuleFromGoModule(pkg.PkgPath).Get() - if !ok { - return optional.None[ParseResult](), fmt.Errorf("package %q is not in the ftl namespace", pkg.PkgPath) - } if len(strings.Split(pkg.PkgPath, "/")) > 2 { // skip subpackages of a module continue } - module.Name = moduleName - if len(pkg.Errors) > 0 { - for _, perr := range pkg.Errors { - merr = append(merr, perr) - } - } - pctx := newParseContext(pkg, pkgs, module, sch) + pctx := newParseContext(pkg, pkgs, sch, out) err := extractInitialDecls(pctx) if err != nil { - return optional.None[ParseResult](), err + return err } for _, file := range pkg.Syntax { err := goast.Visit(file, func(stack []ast.Node, next func() error) (err error) { @@ -166,27 +150,14 @@ func ExtractModuleSchema(dir string, sch *schema.Schema) (optional.Option[ParseR return next() }) if err != nil { - return optional.None[ParseResult](), err + return err } } - for decl, nativeName := range pctx.nativeNames { - nativeNames[decl] = nativeName - } if len(pctx.errors) > 0 { - schemaErrs = append(schemaErrs, maps.Values(pctx.errors)...) + out.Errors = append(out.Errors, maps.Values(pctx.errors)...) } } - if len(schemaErrs) > 0 { - schema.SortErrorsByPosition(schemaErrs) - return optional.Some(ParseResult{Errors: schemaErrs}), nil - } - if len(merr) > 0 { - return optional.None[ParseResult](), errors.Join(merr...) - } - return optional.Some(ParseResult{ - NativeNames: nativeNames, - Module: module, - }), schema.ValidateModule(module) + return nil } // extractInitialDecls traverses the package's AST and extracts declarations needed up front (type aliases, enums and topics) @@ -291,18 +262,7 @@ func extractTypeDecl(pctx *parseContext, node *ast.GenDecl) { } } foundDeclType = optional.Some("enum") - case *directiveTypeAlias: - alias := &schema.TypeAlias{ - Pos: goPosToSchemaPos(node.Pos()), - Comments: parseComments(node.Doc), - Name: strcase.ToUpperCamel(t.Name.Name), - Export: dir.IsExported(), - Type: nil, // nil until next pass, when we can visit the full type graph - } - pctx.module.Decls = append(pctx.module.Decls, alias) - pctx.nativeNames[alias] = nativeName - foundDeclType = optional.Some("type alias") - case *directiveData, *directiveIngress, *directiveVerb, *directiveCronJob, *directiveRetry, *directiveExport, *directiveSubscriber: + case *directiveTypeAlias, *directiveData, *directiveIngress, *directiveVerb, *directiveCronJob, *directiveRetry, *directiveExport, *directiveSubscriber: continue } if foundDeclType, ok := foundDeclType.Get(); ok { @@ -852,7 +812,7 @@ func visitGenDecl(pctx *parseContext, node *ast.GenDecl) { for _, dir := range directives { switch dir.(type) { - case *directiveVerb, *directiveData, *directiveEnum, *directiveTypeAlias: + case *directiveVerb, *directiveData, *directiveEnum: if len(node.Specs) != 1 { pctx.errors.add(errorf(node, "error parsing ftl directive: expected "+ "exactly one type declaration")) @@ -893,28 +853,11 @@ func visitGenDecl(pctx *parseContext, node *ast.GenDecl) { pctx.errors.add(errorf(node, "could not find interface for type enum")) } } - } else if _, ok := dir.(*directiveTypeAlias); ok { - decl, ok := pctx.getDeclForTypeName(t.Name.Name).Get() - if !ok { - pctx.errors.add(errorf(node, "could not find type alias declaration for %q", t.Name.Name)) - return - } - typeAlias, ok := decl.(*schema.TypeAlias) - if !ok { - // This case can be reached if a type is both an enum and a typealias. - // Error is already reported in extractTypeDecls - return - } - if sType, ok := visitType(pctx, node.Pos(), typ, isExported).Get(); ok { - typeAlias.Type = sType - } else { - pctx.errors.add(errorf(node, "unsupported type %q for type alias", typ.Underlying())) - } } else { visitType(pctx, node.Pos(), pctx.pkg.TypesInfo.Defs[t.Name].Type(), isExported) } } - case *directiveIngress, *directiveCronJob, *directiveRetry, *directiveExport, *directiveSubscriber: + case *directiveIngress, *directiveCronJob, *directiveRetry, *directiveExport, *directiveSubscriber, *directiveTypeAlias: } } return @@ -1680,14 +1623,19 @@ type parseContext struct { schema *schema.Schema } -func newParseContext(pkg *packages.Package, pkgs []*packages.Package, module *schema.Module, sch *schema.Schema) *parseContext { +func newParseContext(pkg *packages.Package, pkgs []*packages.Package, sch *schema.Schema, out *analyzers.ExtractResult) *parseContext { + errs := errorSet{} + errs.addAll(out.Errors...) + if out.NativeNames == nil { + out.NativeNames = NativeNames{} + } return &parseContext{ pkg: pkg, pkgs: pkgs, - module: module, - nativeNames: NativeNames{}, + module: out.Module, + nativeNames: out.NativeNames, enumInterfaces: enumInterfaces{}, - errors: errorSet{}, + errors: errs, schema: sch, } } diff --git a/go-runtime/compile/schema_test.go b/go-runtime/compile/schema_test.go index 5fd147aac4..60059a24f6 100644 --- a/go-runtime/compile/schema_test.go +++ b/go-runtime/compile/schema_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/TBD54566975/ftl/go-runtime/schema/analyzers" "github.com/alecthomas/assert/v2" "github.com/alecthomas/participle/v2/lexer" "golang.org/x/tools/go/packages" @@ -448,7 +449,7 @@ func TestParsedirectives(t *testing.T) { func TestParseTypesTime(t *testing.T) { timeRef := mustLoadRef("time", "Time").Type() - pctx := newParseContext(nil, []*packages.Package{}, &schema.Module{}, &schema.Schema{}) + pctx := newParseContext(nil, []*packages.Package{}, &schema.Schema{}, &analyzers.ExtractResult{Module: &schema.Module{}}) parsed, ok := visitType(pctx, token.NoPos, timeRef, false).Get() assert.True(t, ok) _, ok = parsed.(*schema.Time) diff --git a/go-runtime/schema/extractor.go b/go-runtime/schema/extractor.go index b0af02ce8f..c8da4b762a 100644 --- a/go-runtime/schema/extractor.go +++ b/go-runtime/schema/extractor.go @@ -23,7 +23,7 @@ func Extract(moduleDir string) (optional.Option[analyzers.ExtractResult], error) if err != nil { return optional.None[analyzers.ExtractResult](), err } - fResult, ok := (*results)[analyzers.Finalizer] + fResult, ok := results[analyzers.Finalizer] if !ok { return optional.None[analyzers.ExtractResult](), nil }