From 63d34ffb2d5ae3796d60a0fdf433d5d4c4ff42d7 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Thu, 8 Aug 2024 07:26:17 -0400 Subject: [PATCH] make lexgen more independent tool again move packages to file for command line setting mkdir -p outdir --- Makefile | 10 +++---- cmd/lexgen/bsky.json | 26 ++++++++++++++++ cmd/lexgen/main.go | 43 +++++++++++++++++++++++++-- lex/gen.go | 71 +++++++++++++++++++++----------------------- lex/gen_test.go | 19 ++++++++++++ 5 files changed, 125 insertions(+), 44 deletions(-) create mode 100644 cmd/lexgen/bsky.json create mode 100644 lex/gen_test.go diff --git a/Makefile b/Makefile index d6d51bd82..432c71ad0 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ build: ## Build all executables go build ./cmd/fakermaker go build ./cmd/hepa go build ./cmd/supercollider - go build -o ./sonar-cli ./cmd/sonar + go build -o ./sonar-cli ./cmd/sonar go build ./cmd/palomar .PHONY: all @@ -67,7 +67,7 @@ check: ## Compile everything, checking syntax (does not output binaries) .PHONY: lexgen lexgen: ## Run codegen tool for lexicons (lexicon JSON to Go packages) - go run ./cmd/lexgen/ $(LEXDIR) + go run ./cmd/lexgen/ --build-file cmd/lexgen/bsky.json $(LEXDIR) .PHONY: cborgen cborgen: ## Run codegen tool for CBOR serialization @@ -87,8 +87,8 @@ run-dev-opensearch: .env ## Runs a local opensearch instance .PHONY: run-dev-relay run-dev-relay: .env ## Runs 'bigsky' Relay for local dev - GOLOG_LOG_LEVEL=info go run ./cmd/bigsky --admin-key localdev -# --crawl-insecure-ws + GOLOG_LOG_LEVEL=info go run ./cmd/bigsky --admin-key localdev +# --crawl-insecure-ws .PHONY: build-relay-image build-relay-image: ## Builds 'bigsky' Relay docker image @@ -103,7 +103,7 @@ build-relay-ui: ## Build Relay dash web app .PHONY: run-relay-image run-relay-image: docker run -p 2470:2470 bigsky /bigsky --admin-key localdev -# --crawl-insecure-ws +# --crawl-insecure-ws .PHONY: run-dev-search run-dev-search: .env ## Runs search daemon for local dev diff --git a/cmd/lexgen/bsky.json b/cmd/lexgen/bsky.json new file mode 100644 index 000000000..18d1f866e --- /dev/null +++ b/cmd/lexgen/bsky.json @@ -0,0 +1,26 @@ +[ + { + "package": "bsky", + "prefix": "app.bsky", + "outdir": "api/bsky", + "import": "github.com/bluesky-social/indigo/api/bsky" + }, + { + "package": "atproto", + "prefix": "com.atproto", + "outdir": "api/atproto", + "import": "github.com/bluesky-social/indigo/api/atproto" + }, + { + "package": "chat", + "prefix": "chat.bsky", + "outdir": "api/chat", + "import": "github.com/bluesky-social/indigo/api/chat" + }, + { + "package": "ozone", + "prefix": "tools.ozone", + "outdir": "api/ozone", + "import": "github.com/bluesky-social/indigo/api/ozone" + } +] diff --git a/cmd/lexgen/main.go b/cmd/lexgen/main.go index 48bd55bda..47f8a8a16 100644 --- a/cmd/lexgen/main.go +++ b/cmd/lexgen/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "io/fs" "os" @@ -79,6 +80,14 @@ func main() { Name: "package", Value: "schemagen", }, + &cli.StringFlag{ + Name: "build", + Value: "", + }, + &cli.StringFlag{ + Name: "build-file", + Value: "", + }, } app.Action = func(cctx *cli.Context) error { paths, err := expandArgs(cctx.Args().Slice()) @@ -100,13 +109,43 @@ func main() { schemas = append(schemas, s) } + buildLiteral := cctx.String("build") + buildPath := cctx.String("build-file") + var packages []lex.Package + if buildLiteral != "" { + if buildPath != "" { + return errors.New("must not set both --build and --build-file") + } + packages, err = lex.ParsePackages([]byte(buildLiteral)) + if err != nil { + return fmt.Errorf("--build error, %w", err) + } + if len(packages) == 0 { + return errors.New("--build must specify at least one Package{}") + } + } else if buildPath != "" { + blob, err := os.ReadFile(buildPath) + if err != nil { + return fmt.Errorf("--build-file error, %w", err) + } + packages, err = lex.ParsePackages(blob) + if err != nil { + return fmt.Errorf("--build-file error, %w", err) + } + if len(packages) == 0 { + return errors.New("--build-file must specify at least one Package{}") + } + } else { + return errors.New("need exactly one of --build or --build-file") + } + if cctx.Bool("gen-server") { pkgname := cctx.String("package") outdir := cctx.String("outdir") if outdir == "" { return fmt.Errorf("must specify output directory (--outdir)") } - defmap := lex.BuildExtDefMap(schemas) + defmap := lex.BuildExtDefMap(schemas, packages) _ = defmap paths := cctx.StringSlice("types-import") @@ -123,7 +162,7 @@ func main() { } } else { - return lex.Run(schemas) + return lex.Run(schemas, packages) } return nil diff --git a/lex/gen.go b/lex/gen.go index e5777dd82..4d38152fd 100644 --- a/lex/gen.go +++ b/lex/gen.go @@ -229,7 +229,7 @@ func ReadSchema(f string) (*Schema, error) { // Build total map of all types defined inside schemas. // Return map from fully qualified type name to its *TypeSchema -func BuildExtDefMap(ss []*Schema) map[string]*ExtDef { +func BuildExtDefMap(ss []*Schema, packages []Package) map[string]*ExtDef { out := make(map[string]*ExtDef) for _, s := range ss { for k, d := range s.Defs { @@ -238,7 +238,7 @@ func BuildExtDefMap(ss []*Schema) map[string]*ExtDef { d.defName = k var pref string - for _, pkg := range Packages { + for _, pkg := range packages { if strings.HasPrefix(s.ID, pkg.Prefix) { pref = pkg.Prefix break @@ -303,20 +303,25 @@ func printerf(w io.Writer) func(format string, args ...any) { } } -func GenCodeForSchema(pkg string, prefix string, fname string, reqcode bool, s *Schema, defmap map[string]*ExtDef) error { +func GenCodeForSchema(pkg Package, reqcode bool, s *Schema, packages []Package, defmap map[string]*ExtDef) error { + err := os.MkdirAll(pkg.Outdir, 0755) + if err != nil { + return fmt.Errorf("%s: could not mkdir, %w", pkg.Outdir, err) + } + fname := filepath.Join(pkg.Outdir, s.Name()+".go") buf := new(bytes.Buffer) pf := printerf(buf) - s.prefix = prefix + s.prefix = pkg.Prefix for _, d := range s.Defs { - d.prefix = prefix + d.prefix = pkg.Prefix } // Add the standard Go generated code header as recognized by GitHub, VS Code, etc. // See https://golang.org/s/generatedcode. pf("// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.\n\n") - pf("package %s\n\n", pkg) + pf("package %s\n\n", pkg.GoPackage) pf("// schema: %s\n\n", s.ID) @@ -327,14 +332,14 @@ func GenCodeForSchema(pkg string, prefix string, fname string, reqcode bool, s * pf("\tcbg \"github.com/whyrusleeping/cbor-gen\"\n") pf("\t\"github.com/bluesky-social/indigo/xrpc\"\n") pf("\t\"github.com/bluesky-social/indigo/lex/util\"\n") - for k, v := range prefixToGoImport { - if k != prefix { - pf("\t%s %q\n", importNameForPrefix(k), v) + for _, xpkg := range packages { + if xpkg.Prefix != pkg.Prefix { + pf("\t%s %q\n", importNameForPrefix(xpkg.Prefix), xpkg.Import) } } pf(")\n\n") - tps := s.AllTypes(prefix, defmap) + tps := s.AllTypes(pkg.Prefix, defmap) if err := writeDecoderRegister(buf, tps); err != nil { return err @@ -350,8 +355,9 @@ func GenCodeForSchema(pkg string, prefix string, fname string, reqcode bool, s * } } + // reqcode is always True if reqcode { - name := nameFromID(s.ID, prefix) + name := nameFromID(s.ID, pkg.Prefix) main, ok := s.Defs["main"] if ok { if err := writeMethods(name, main, buf); err != nil { @@ -398,7 +404,7 @@ func writeCodeFile(b []byte, fname string) error { if werr != nil { return werr } - return fmt.Errorf("failed to format output of %q with goimports: %w", fname, err) + return fmt.Errorf("failed to format output of %q with goimports: %w (wrote failed file to ./temp)", fname, err) } if err := os.WriteFile(fname, fixed, 0664); err != nil { @@ -1475,46 +1481,37 @@ func (ts *TypeSchema) writeCborUnmarshalerEnum(name string, w io.Writer) error { } type Package struct { - GoPackage string - Prefix string - Outdir string -} - -var Packages []Package = []Package{ - Package{"bsky", "app.bsky", "api/bsky"}, - Package{"atproto", "com.atproto", "api/atproto"}, - Package{"chat", "chat.bsky", "api/chat"}, - Package{"ozone", "tools.ozone", "api/ozone"}, + GoPackage string `json:"package"` + Prefix string `json:"prefix"` + Outdir string `json:"outdir"` + Import string `json:"import"` } -var prefixToGoImport map[string]string - -func init() { - prefixToGoImport = map[string]string{ - "app.bsky": "github.com/bluesky-social/indigo/api/bsky", - "chat.bsky": "github.com/bluesky-social/indigo/api/chat", - "com.atproto": "github.com/bluesky-social/indigo/api/atproto", - "tools.ozone": "github.com/bluesky-social/indigo/api/ozone", +// ParsePackages reads a json blob which should be an array of Package{} objects. +func ParsePackages(jsonBytes []byte) ([]Package, error) { + var packages []Package + err := json.Unmarshal(jsonBytes, &packages) + if err != nil { + return nil, err } + return packages, nil } -func Run(schemas []*Schema) error { - defmap := BuildExtDefMap(schemas) +func Run(schemas []*Schema, packages []Package) error { + defmap := BuildExtDefMap(schemas, packages) - for _, pkg := range Packages { + for _, pkg := range packages { prefix := pkg.Prefix FixRecordReferences(schemas, defmap, prefix) } - for _, pkg := range Packages { + for _, pkg := range packages { for _, s := range schemas { if !strings.HasPrefix(s.ID, pkg.Prefix) { continue } - fname := filepath.Join(pkg.Outdir, s.Name()+".go") - - if err := GenCodeForSchema(pkg.GoPackage, pkg.Prefix, fname, true, s, defmap); err != nil { + if err := GenCodeForSchema(pkg, true, s, packages, defmap); err != nil { return fmt.Errorf("failed to process schema %q: %w", s.path, err) } } diff --git a/lex/gen_test.go b/lex/gen_test.go new file mode 100644 index 000000000..5544fe42c --- /dev/null +++ b/lex/gen_test.go @@ -0,0 +1,19 @@ +package lex + +import "testing" + +func TestParsePackages(t *testing.T) { + text := `[{"package": "bsky", "prefix": "app.bsky", "outdir": "api/bsky", "import": "github.com/bluesky-social/indigo/api/bsky"}]` + parsed, err := ParsePackages([]byte(text)) + if err != nil { + t.Fatalf("error parsing json: %s", err) + } + if len(parsed) != 1 { + t.Fatalf("expected 1, got %d", len(parsed)) + } + expected := Package{"bsky", "app.bsky", "api/bsky", "github.com/bluesky-social/indigo/api/bsky"} + if expected != parsed[0] { + t.Fatalf("expected %#v, got %#v", expected, parsed[0]) + } + +}