make lexgen more independent tool again

move packages to file for command line setting
mkdir -p outdir

Makefile

@@ -25,7 +25,7 @@ build: ## Build all executables
 	go build ./cmd/fakermaker
 	go build ./cmd/hepa
 	go build ./cmd/supercollider
-	go build -o ./sonar-cli ./cmd/sonar 
+	go build -o ./sonar-cli ./cmd/sonar
 	go build ./cmd/palomar
 
 .PHONY: all
@@ -67,7 +67,7 @@ check: ## Compile everything, checking syntax (does not output binaries)
 
 .PHONY: lexgen
 lexgen: ## Run codegen tool for lexicons (lexicon JSON to Go packages)
-	go run ./cmd/lexgen/ $(LEXDIR)
+	go run ./cmd/lexgen/ --build-file cmd/lexgen/bsky.json $(LEXDIR)
 
 .PHONY: cborgen
 cborgen: ## Run codegen tool for CBOR serialization
@@ -87,8 +87,8 @@ run-dev-opensearch: .env ## Runs a local opensearch instance
 
 .PHONY: run-dev-relay
 run-dev-relay: .env ## Runs 'bigsky' Relay for local dev
-	GOLOG_LOG_LEVEL=info go run ./cmd/bigsky --admin-key localdev 
-# --crawl-insecure-ws 
+	GOLOG_LOG_LEVEL=info go run ./cmd/bigsky --admin-key localdev
+# --crawl-insecure-ws
 
 .PHONY: build-relay-image
 build-relay-image: ## Builds 'bigsky' Relay docker image
@@ -103,7 +103,7 @@ build-relay-ui: ## Build Relay dash web app
 .PHONY: run-relay-image
 run-relay-image:
 	docker run -p 2470:2470 bigsky /bigsky --admin-key localdev
-# --crawl-insecure-ws 
+# --crawl-insecure-ws
 
 .PHONY: run-dev-search
 run-dev-search: .env ## Runs search daemon for local dev

cmd/lexgen/bsky.json

@@ -0,0 +1,26 @@
+[
+  {
+    "package": "bsky",
+    "prefix": "app.bsky",
+    "outdir": "api/bsky",
+    "import": "github.com/bluesky-social/indigo/api/bsky"
+  },
+  {
+    "package": "atproto",
+    "prefix": "com.atproto",
+    "outdir": "api/atproto",
+    "import": "github.com/bluesky-social/indigo/api/atproto"
+  },
+  {
+    "package": "chat",
+    "prefix": "chat.bsky",
+    "outdir": "api/chat",
+    "import": "github.com/bluesky-social/indigo/api/chat"
+  },
+  {
+    "package": "ozone",
+    "prefix": "tools.ozone",
+    "outdir": "api/ozone",
+    "import": "github.com/bluesky-social/indigo/api/ozone"
+  }
+]

cmd/lexgen/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -79,6 +80,14 @@ func main() {
 			Name:  "package",
 			Value: "schemagen",
 		},
+		&cli.StringFlag{
+			Name:  "build",
+			Value: "",
+		},
+		&cli.StringFlag{
+			Name:  "build-file",
+			Value: "",
+		},
 	}
 	app.Action = func(cctx *cli.Context) error {
 		paths, err := expandArgs(cctx.Args().Slice())
@@ -100,13 +109,43 @@ func main() {
 			schemas = append(schemas, s)
 		}
 
+		buildLiteral := cctx.String("build")
+		buildPath := cctx.String("build-file")
+		var packages []lex.Package
+		if buildLiteral != "" {
+			if buildPath != "" {
+				return errors.New("must not set both --build and --build-file")
+			}
+			packages, err = lex.ParsePackages([]byte(buildLiteral))
+			if err != nil {
+				return fmt.Errorf("--build error, %w", err)
+			}
+			if len(packages) == 0 {
+				return errors.New("--build must specify at least one Package{}")
+			}
+		} else if buildPath != "" {
+			blob, err := os.ReadFile(buildPath)
+			if err != nil {
+				return fmt.Errorf("--build-file error, %w", err)
+			}
+			packages, err = lex.ParsePackages(blob)
+			if err != nil {
+				return fmt.Errorf("--build-file error, %w", err)
+			}
+			if len(packages) == 0 {
+				return errors.New("--build-file must specify at least one Package{}")
+			}
+		} else {
+			return errors.New("need exactly one of --build or --build-file")
+		}
+
 		if cctx.Bool("gen-server") {
 			pkgname := cctx.String("package")
 			outdir := cctx.String("outdir")
 			if outdir == "" {
 				return fmt.Errorf("must specify output directory (--outdir)")
 			}
-			defmap := lex.BuildExtDefMap(schemas)
+			defmap := lex.BuildExtDefMap(schemas, packages)
 			_ = defmap
 
 			paths := cctx.StringSlice("types-import")
@@ -123,7 +162,7 @@ func main() {
 			}
 
 		} else {
-			return lex.Run(schemas)
+			return lex.Run(schemas, packages)
 		}
 
 		return nil

lex/gen.go

@@ -229,7 +229,7 @@ func ReadSchema(f string) (*Schema, error) {
 
 // Build total map of all types defined inside schemas.
 // Return map from fully qualified type name to its *TypeSchema
-func BuildExtDefMap(ss []*Schema) map[string]*ExtDef {
+func BuildExtDefMap(ss []*Schema, packages []Package) map[string]*ExtDef {
 	out := make(map[string]*ExtDef)
 	for _, s := range ss {
 		for k, d := range s.Defs {
@@ -238,7 +238,7 @@ func BuildExtDefMap(ss []*Schema) map[string]*ExtDef {
 			d.defName = k
 
 			var pref string
-			for _, pkg := range Packages {
+			for _, pkg := range packages {
 				if strings.HasPrefix(s.ID, pkg.Prefix) {
 					pref = pkg.Prefix
 					break
@@ -303,20 +303,25 @@ func printerf(w io.Writer) func(format string, args ...any) {
 	}
 }
 
-func GenCodeForSchema(pkg string, prefix string, fname string, reqcode bool, s *Schema, defmap map[string]*ExtDef) error {
+func GenCodeForSchema(pkg Package, reqcode bool, s *Schema, packages []Package, defmap map[string]*ExtDef) error {
+	err := os.MkdirAll(pkg.Outdir, 0755)
+	if err != nil {
+		return fmt.Errorf("%s: could not mkdir, %w", pkg.Outdir, err)
+	}
+	fname := filepath.Join(pkg.Outdir, s.Name()+".go")
 	buf := new(bytes.Buffer)
 	pf := printerf(buf)
 
-	s.prefix = prefix
+	s.prefix = pkg.Prefix
 	for _, d := range s.Defs {
-		d.prefix = prefix
+		d.prefix = pkg.Prefix
 	}
 
 	// Add the standard Go generated code header as recognized by GitHub, VS Code, etc.
 	// See https://golang.org/s/generatedcode.
 	pf("// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT.\n\n")
 
-	pf("package %s\n\n", pkg)
+	pf("package %s\n\n", pkg.GoPackage)
 
 	pf("// schema: %s\n\n", s.ID)
 
@@ -327,14 +332,14 @@ func GenCodeForSchema(pkg string, prefix string, fname string, reqcode bool, s *
 	pf("\tcbg \"github.com/whyrusleeping/cbor-gen\"\n")
 	pf("\t\"github.com/bluesky-social/indigo/xrpc\"\n")
 	pf("\t\"github.com/bluesky-social/indigo/lex/util\"\n")
-	for k, v := range prefixToGoImport {
-		if k != prefix {
-			pf("\t%s %q\n", importNameForPrefix(k), v)
+	for _, xpkg := range packages {
+		if xpkg.Prefix != pkg.Prefix {
+			pf("\t%s %q\n", importNameForPrefix(xpkg.Prefix), xpkg.Import)
 		}
 	}
 	pf(")\n\n")
 
-	tps := s.AllTypes(prefix, defmap)
+	tps := s.AllTypes(pkg.Prefix, defmap)
 
 	if err := writeDecoderRegister(buf, tps); err != nil {
 		return err
@@ -350,8 +355,9 @@ func GenCodeForSchema(pkg string, prefix string, fname string, reqcode bool, s *
 		}
 	}
 
+	// reqcode is always True
 	if reqcode {
-		name := nameFromID(s.ID, prefix)
+		name := nameFromID(s.ID, pkg.Prefix)
 		main, ok := s.Defs["main"]
 		if ok {
 			if err := writeMethods(name, main, buf); err != nil {
@@ -398,7 +404,7 @@ func writeCodeFile(b []byte, fname string) error {
 		if werr != nil {
 			return werr
 		}
-		return fmt.Errorf("failed to format output of %q with goimports: %w", fname, err)
+		return fmt.Errorf("failed to format output of %q with goimports: %w (wrote failed file to ./temp)", fname, err)
 	}
 
 	if err := os.WriteFile(fname, fixed, 0664); err != nil {
@@ -1475,46 +1481,37 @@ func (ts *TypeSchema) writeCborUnmarshalerEnum(name string, w io.Writer) error {
 }
 
 type Package struct {
-	GoPackage string
-	Prefix    string
-	Outdir    string
-}
-
-var Packages []Package = []Package{
-	Package{"bsky", "app.bsky", "api/bsky"},
-	Package{"atproto", "com.atproto", "api/atproto"},
-	Package{"chat", "chat.bsky", "api/chat"},
-	Package{"ozone", "tools.ozone", "api/ozone"},
+	GoPackage string `json:"package"`
+	Prefix    string `json:"prefix"`
+	Outdir    string `json:"outdir"`
+	Import    string `json:"import"`
 }
 
-var prefixToGoImport map[string]string
-
-func init() {
-	prefixToGoImport = map[string]string{
-		"app.bsky":    "github.com/bluesky-social/indigo/api/bsky",
-		"chat.bsky":   "github.com/bluesky-social/indigo/api/chat",
-		"com.atproto": "github.com/bluesky-social/indigo/api/atproto",
-		"tools.ozone": "github.com/bluesky-social/indigo/api/ozone",
+// ParsePackages reads a json blob which should be an array of Package{} objects.
+func ParsePackages(jsonBytes []byte) ([]Package, error) {
+	var packages []Package
+	err := json.Unmarshal(jsonBytes, &packages)
+	if err != nil {
+		return nil, err
 	}
+	return packages, nil
 }
 
-func Run(schemas []*Schema) error {
-	defmap := BuildExtDefMap(schemas)
+func Run(schemas []*Schema, packages []Package) error {
+	defmap := BuildExtDefMap(schemas, packages)
 
-	for _, pkg := range Packages {
+	for _, pkg := range packages {
 		prefix := pkg.Prefix
 		FixRecordReferences(schemas, defmap, prefix)
 	}
 
-	for _, pkg := range Packages {
+	for _, pkg := range packages {
 		for _, s := range schemas {
 			if !strings.HasPrefix(s.ID, pkg.Prefix) {
 				continue
 			}
 
-			fname := filepath.Join(pkg.Outdir, s.Name()+".go")
-
-			if err := GenCodeForSchema(pkg.GoPackage, pkg.Prefix, fname, true, s, defmap); err != nil {
+			if err := GenCodeForSchema(pkg, true, s, packages, defmap); err != nil {
 				return fmt.Errorf("failed to process schema %q: %w", s.path, err)
 			}
 		}

lex/gen_test.go

@@ -0,0 +1,19 @@
+package lex
+
+import "testing"
+
+func TestParsePackages(t *testing.T) {
+	text := `[{"package": "bsky", "prefix": "app.bsky", "outdir": "api/bsky", "import": "github.com/bluesky-social/indigo/api/bsky"}]`
+	parsed, err := ParsePackages([]byte(text))
+	if err != nil {
+		t.Fatalf("error parsing json: %s", err)
+	}
+	if len(parsed) != 1 {
+		t.Fatalf("expected 1, got %d", len(parsed))
+	}
+	expected := Package{"bsky", "app.bsky", "api/bsky", "github.com/bluesky-social/indigo/api/bsky"}
+	if expected != parsed[0] {
+		t.Fatalf("expected %#v, got %#v", expected, parsed[0])
+	}
+
+}