Add positional information to AST Nodes (#5108)

brimdata · Apr 29, 2024 · 183e432 · 183e432
1 parent 9c53200
commit 183e432
Show file tree

Hide file tree

Showing 19 changed files with 5,019 additions and 3,895 deletions.
diff --git a/compiler/ast/ast.go b/compiler/ast/ast.go
diff --git a/compiler/ast/zed/type.go b/compiler/ast/zed/type.go
@@ -1,58 +1,80 @@
 package zed
 
 type Type interface {
+	Node
 	typeNode()
 }
 
+type Node interface {
+	Pos() int
+	End() int
+}
+
 type (
 	TypePrimitive struct {
-		Kind string `json:"kind" unpack:""`
-		Name string `json:"name"`
+		Kind    string `json:"kind" unpack:""`
+		Name    string `json:"name"`
+		NamePos int    `json:"name_pos"`
 	}
 	TypeRecord struct {
 		Kind   string      `json:"kind" unpack:""`
+		Lbrace int         `json:"lbrace"`
 		Fields []TypeField `json:"fields"`
+		Rbrace int         `json:"rbrace"`
 	}
 	TypeField struct {
 		Name string `json:"name"`
 		Type Type   `json:"type"`
 	}
 	TypeArray struct {
-		Kind string `json:"kind" unpack:""`
-		Type Type   `json:"type"`
+		Kind   string `json:"kind" unpack:""`
+		Lbrack int    `json:"lbrack"`
+		Type   Type   `json:"type"`
+		Rbrack int    `json:"rbrack"`
 	}
 	TypeSet struct {
-		Kind string `json:"kind" unpack:""`
-		Type Type   `json:"type"`
+		Kind  string `json:"kind" unpack:""`
+		Lpipe int    `json:"lpipe"`
+		Type  Type   `json:"type"`
+		Rpipe int    `json:"rpipe"`
 	}
 	TypeUnion struct {
-		Kind  string `json:"kind" unpack:""`
-		Types []Type `json:"types"`
+		Kind   string `json:"kind" unpack:""`
+		Lparen int    `json:"lparen"`
+		Types  []Type `json:"types"`
+		Rparen int    `json:"rparen"`
 	}
 	TypeEnum struct {
 		Kind    string   `json:"kind" unpack:""`
 		Symbols []string `json:"symbols"`
 	}
 	TypeMap struct {
 		Kind    string `json:"kind" unpack:""`
+		Lpipe   int    `json:"lpipe"`
 		KeyType Type   `json:"key_type"`
 		ValType Type   `json:"val_type"`
+		Rpipe   int    `json:"rpipe"`
 	}
 	TypeNull struct {
-		Kind string `json:"kind" unpack:""`
+		Kind       string `json:"kind" unpack:""`
+		KeywordPos int    `json:"pos"`
 	}
 	TypeError struct {
-		Kind string `json:"kind" unpack:""`
-		Type Type   `json:"type"`
+		Kind       string `json:"kind" unpack:""`
+		KeywordPos int    `json:"keyword_pos"`
+		Type       Type   `json:"type"`
+		Rparen     int    `json:"rparen"`
 	}
 	TypeName struct {
-		Kind string `json:"kind" unpack:""`
-		Name string `json:"name"`
+		Kind    string `json:"kind" unpack:""`
+		Name    string `json:"name"`
+		NamePos int    `json:"name_pos"`
 	}
 	TypeDef struct {
-		Kind string `json:"kind" unpack:""`
-		Name string `json:"name"`
-		Type Type   `json:"type"`
+		Kind    string `json:"kind" unpack:""`
+		Name    string `json:"name"`
+		NamePos int    `json:"name_pos"`
+		Type    Type   `json:"type"`
 	}
 )
 
@@ -67,3 +89,27 @@ func (*TypeNull) typeNode()      {}
 func (*TypeError) typeNode()     {}
 func (*TypeName) typeNode()      {}
 func (*TypeDef) typeNode()       {}
+
+func (x *TypePrimitive) Pos() int { return x.NamePos }
+func (x *TypeRecord) Pos() int    { return x.Lbrace }
+func (x *TypeArray) Pos() int     { return x.Lbrack }
+func (x *TypeSet) Pos() int       { return x.Lpipe }
+func (x *TypeUnion) Pos() int     { return x.Lparen }
+func (x *TypeEnum) Pos() int      { return -1 } // TypeEnum isn't supported in Zed language
+func (x *TypeMap) Pos() int       { return x.Lpipe }
+func (x *TypeNull) Pos() int      { return x.KeywordPos }
+func (x *TypeError) Pos() int     { return x.KeywordPos }
+func (x *TypeName) Pos() int      { return x.NamePos }
+func (x *TypeDef) Pos() int       { return x.NamePos }
+
+func (x *TypePrimitive) End() int { return x.NamePos + len(x.Name) }
+func (x *TypeRecord) End() int    { return x.Rbrace + 1 }
+func (x *TypeArray) End() int     { return x.Rbrack + 1 }
+func (x *TypeSet) End() int       { return x.Rpipe + 1 }
+func (x *TypeUnion) End() int     { return x.Rparen + 1 }
+func (x *TypeEnum) End() int      { return -1 } // TypeEnum isn't supported in Zed language
+func (x *TypeMap) End() int       { return x.Rpipe + 1 }
+func (x *TypeNull) End() int      { return x.KeywordPos + 4 }
+func (x *TypeError) End() int     { return x.Rparen + 1 }
+func (x *TypeName) End() int      { return x.NamePos + len(x.Name) }
+func (x *TypeDef) End() int       { return x.Type.End() }
diff --git a/compiler/ast/zed/value.go b/compiler/ast/zed/value.go
@@ -29,14 +29,18 @@ type Any interface {
 	anyNode()
 }
 
+// Only Primitive and TypeValue are used in zed parser so only these are given
+// ast.Node features.
+
 type (
 	Primitive struct {
-		Kind string `json:"kind" unpack:""`
-		Type string `json:"type"`
-		Text string `json:"text"`
+		Kind    string `json:"kind" unpack:""`
+		Type    string `json:"type"`
+		Text    string `json:"text"`
+		TextPos int    `json:"text_pos"`
 	}
 	Record struct {
-		Kind   string  `json:"Kind" unpack:""`
+		Kind   string  `json:"kind" unpack:""`
 		Fields []Field `json:"fields"`
 	}
 	Field struct {
@@ -64,8 +68,10 @@ type (
 		Value Value `json:"value"`
 	}
 	TypeValue struct {
-		Kind  string `json:"kind" unpack:""`
-		Value Type   `json:"value"`
+		Kind   string `json:"kind" unpack:""`
+		Lbrack int    `json:"lbrack"`
+		Value  Type   `json:"value"`
+		Rbrack int    `json:"rbrack"`
 	}
 	Error struct {
 		Kind  string `json:"kind" unpack:""`
@@ -87,3 +93,9 @@ func (*TypeValue) ExprAST() {}
 
 func (*Primitive) ExprDAG() {}
 func (*TypeValue) ExprDAG() {}
+
+func (x *Primitive) Pos() int { return x.TextPos }
+func (x *TypeValue) Pos() int { return x.Lbrack }
+
+func (x *Primitive) End() int { return x.TextPos + len(x.Text) }
+func (x *TypeValue) End() int { return x.Rbrack + 1 }
diff --git a/compiler/job.go b/compiler/job.go
@@ -89,11 +89,7 @@ func (j *Job) Parallelize(n int) error {
 }
 
 func Parse(src string, filenames ...string) (ast.Seq, error) {
-	parsed, err := parser.ParseZed(filenames, src)
-	if err != nil {
-		return nil, err
-	}
-	return ast.UnmarshalObject(parsed)
+	return parser.ParseZed(filenames, src)
 }
 
 // MustParse is like Parse but panics if an error is encountered.

diff --git a/compiler/parser/Makefile b/compiler/parser/Makefile
@@ -1,9 +1,6 @@
 GOIMPORTS_VERSION = v0.19.0
 PIGEON_VERSION = v1.2.1
 
-# GNU cpp needs -std=gnu90 to prevent errors on \uHHHH sequences, and Clang cpp
-# with -std=gnu90 needs -Wno-unicode to prevent warnings on \uHHHH sequences.
-cpp = cpp -E -P -std=gnu90 -Wno-unicode
 deps = $(CURDIR)/deps
 
 all: parser.go
@@ -16,5 +13,5 @@ endif
 ifeq "$(shell go version -m $(deps)/bin/pigeon 2>&1 | fgrep $(PIGEON_VERSION))" ""
 	GOBIN=$(deps)/bin go install github.com/mna/pigeon@$(PIGEON_VERSION)
 endif
-	$(cpp) -DGO parser.peg | $(deps)/bin/pigeon -support-left-recursion -o $@
+	$(deps)/bin/pigeon -support-left-recursion -o $@ parser.peg
 	$(deps)/bin/goimports -w $@
diff --git a/compiler/parser/api.go b/compiler/parser/api.go
@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"os"
 	"strings"
+
+	"github.com/brimdata/zed/compiler/ast"
 )
 
 // ParseZed calls ConcatSource followed by Parse.  If Parse fails, it calls
 // ImproveError.
-func ParseZed(filenames []string, src string) (interface{}, error) {
+func ParseZed(filenames []string, src string) (ast.Seq, error) {
 	src, srcInfo, err := ConcatSource(filenames, src)
 	if err != nil {
 		return nil, err
@@ -17,7 +19,7 @@ func ParseZed(filenames []string, src string) (interface{}, error) {
 	if err != nil {
 		return nil, ImproveError(err, src, srcInfo)
 	}
-	return p, nil
+	return sliceOf[ast.Op](p), nil
 }
 
 // SourceInfo holds source file offsets.