Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improvements to "from" operator #5378

Merged
merged 3 commits into from
Oct 28, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/super/ztests/from-file-error.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
script: |
! super -I query.zed
! super -I query.spq

inputs:
- name: query.zed
- name: query.spq
data: |
file a.zson
file a.jsup

outputs:
- name: stderr
regexp: |
a.zson: file does not exist
a.jsup: file:///.*/a.jsup: file does not exist.*
6 changes: 2 additions & 4 deletions cmd/super/ztests/from-pool-error.yaml
Original file line number Diff line number Diff line change
@@ -3,7 +3,5 @@ script: |

outputs:
- name: stderr
data: |
super: "from pool" cannot be used without a lake at line 1, column 8:
from ( pool a )
~~~~~~
regexp: |
a: file:///.*/a: file does not exist
97 changes: 49 additions & 48 deletions compiler/ast/ast.go
Original file line number Diff line number Diff line change
@@ -144,14 +144,22 @@ type Name struct {
Loc `json:"loc"`
}

type Pattern interface {
type FromEntity interface {
Node
PatternAST()
fromEntityAST()
}

func (*Glob) PatternAST() {}
func (*Regexp) PatternAST() {}
func (*Name) PatternAST() {}
type ExprEntity struct {
Kind string `json:"kind" unpack:""`
Expr Expr `json:"expr"`
Loc `json:"loc"`
}

func (*Glob) fromEntityAST() {}
func (*Regexp) fromEntityAST() {}
func (*ExprEntity) fromEntityAST() {}
func (*LakeMeta) fromEntityAST() {}
func (*Name) fromEntityAST() {}

type RecordExpr struct {
Kind string `json:"kind" unpack:""`
@@ -492,11 +500,6 @@ type (
Kind string `json:"kind" unpack:""`
Loc `json:"loc"`
}
From struct {
Kind string `json:"kind" unpack:""`
Trunks []Seq `json:"trunks"`
Loc `json:"loc"`
}
Load struct {
Kind string `json:"kind" unpack:""`
Pool *Name `json:"pool"`
@@ -524,55 +527,56 @@ type (
}
)

// Source structure

type (
File struct {
Kind string `json:"kind" unpack:""`
Path Pattern `json:"path"`
Format *Name `json:"format"`
SortKeys []SortExpr `json:"sort_keys"`
Loc `json:"loc"`
}
HTTP struct {
Kind string `json:"kind" unpack:""`
URL Pattern `json:"url"`
Format *Name `json:"format"`
SortKeys []SortExpr `json:"sort_keys"`
Method *Name `json:"method"`
Headers *RecordExpr `json:"headers"`
Body *Name `json:"body"`
Loc `json:"loc"`
}
Pool struct {
Kind string `json:"kind" unpack:""`
Spec PoolSpec `json:"spec"`
Loc `json:"loc"`
From struct {
Kind string `json:"kind" unpack:""`
Entity FromEntity `json:"entity"`
Args FromArgs `json:"args"`
Loc `json:"loc"`
}
LakeMeta struct {
Kind string `json:"kind" unpack:""`
MetaPos int `json:"meta_pos"`
Meta *Name `json:"meta"`
Loc `json:"loc"`
}
Delete struct {
Kind string `json:"kind" unpack:""`
Loc `json:"loc"` // dummy field, not needed except to implement Node
}
)

type PoolSpec struct {
Pool Pattern `json:"pool"`
Commit *Name `json:"commit"`
Meta *Name `json:"meta"`
Tap bool `json:"tap"`
type PoolArgs struct {
Kind string `json:"kind" unpack:""`
Commit *Name `json:"commit"`
Meta *Name `json:"meta"`
Tap bool `json:"tap"`
Loc `json:"loc"`
}

type Source interface {
type FormatArg struct {
Kind string `json:"kind" unpack:""`
Format *Name `json:"format"`
Loc `json:"loc"`
}

type HTTPArgs struct {
Kind string `json:"kind" unpack:""`
Format *Name `json:"format"`
Method *Name `json:"method"`
Headers *RecordExpr `json:"headers"`
Body *Name `json:"body"`
Loc `json:"loc"`
}

type FromArgs interface {
Node
Source()
fromArgs()
}

func (*Pool) Source() {}
func (*File) Source() {}
func (*HTTP) Source() {}
func (*Pass) Source() {}
func (*Delete) Source() {}
func (*PoolArgs) fromArgs() {}
func (*FormatArg) fromArgs() {}
func (*HTTPArgs) fromArgs() {}

type SortExpr struct {
Kind string `json:"kind" unpack:""`
@@ -615,9 +619,6 @@ func (*Cut) OpAST() {}
func (*Drop) OpAST() {}
func (*Head) OpAST() {}
func (*Tail) OpAST() {}
func (*Pool) OpAST() {}
func (*File) OpAST() {}
func (*HTTP) OpAST() {}
func (*Pass) OpAST() {}
func (*Uniq) OpAST() {}
func (*Summarize) OpAST() {}
9 changes: 6 additions & 3 deletions compiler/ast/unpack.go
Original file line number Diff line number Diff line change
@@ -27,8 +27,9 @@ var unpacker = unpack.New(
Explode{},
Enum{},
Error{},
ExprEntity{},
FieldExpr{},
File{},
FormatArg{},
From{},
FString{},
FStringExpr{},
@@ -38,7 +39,7 @@ var unpacker = unpack.New(
Summarize{},
Grep{},
Head{},
HTTP{},
HTTPArgs{},
ID{},
ImpliedValue{},
IndexExpr{},
@@ -53,7 +54,7 @@ var unpacker = unpack.New(
OverExpr{},
Parallel{},
Pass{},
Pool{},
PoolArgs{},
Primitive{},
Put{},
Record{},
@@ -94,6 +95,8 @@ var unpacker = unpack.New(
Where{},
Yield{},
Sample{},
Delete{},
LakeMeta{},
)

// UnmarshalOp transforms a JSON representation of an operator into an Op.
22 changes: 10 additions & 12 deletions compiler/dag/op.go
Original file line number Diff line number Diff line change
@@ -206,20 +206,18 @@ type (
SortKeys order.SortKeys `json:"sort_keys"`
}
FileScan struct {
Kind string `json:"kind" unpack:""`
Path string `json:"path"`
Format string `json:"format"`
SortKeys order.SortKeys `json:"sort_keys"`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it's reason enough to keep this (we can just add it back if we ever want it) but Parquet has support for sorted data: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L757-L770

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that the system should be able to use sort keys from file objects but this doesn't belong in the query language. Instead this should be part of the interface between the compiler and the file objects as it is with the pool adaptor.

Filter Expr `json:"filter"`
Kind string `json:"kind" unpack:""`
Path string `json:"path"`
Format string `json:"format"`
Filter Expr `json:"filter"`
}
HTTPScan struct {
Kind string `json:"kind" unpack:""`
URL string `json:"url"`
Format string `json:"format"`
SortKeys order.SortKeys `json:"sort_keys"`
Method string `json:"method"`
Headers map[string][]string `json:"headers"`
Body string `json:"body"`
Kind string `json:"kind" unpack:""`
URL string `json:"url"`
Format string `json:"format"`
Method string `json:"method"`
Headers map[string][]string `json:"headers"`
Body string `json:"body"`
}
PoolScan struct {
Kind string `json:"kind" unpack:""`
4 changes: 2 additions & 2 deletions compiler/optimizer/optimizer.go
Original file line number Diff line number Diff line change
@@ -416,9 +416,9 @@ func (o *Optimizer) sortKeysOfSource(op dag.Op) (order.SortKeys, error) {
case *dag.DefaultScan:
return op.SortKeys, nil
case *dag.FileScan:
return op.SortKeys, nil
return nil, nil
case *dag.HTTPScan:
return op.SortKeys, nil
return nil, nil
case *dag.PoolScan:
return o.sortKey(op.ID)
case *dag.Lister:
5,891 changes: 3,005 additions & 2,886 deletions compiler/parser/parser.go

Large diffs are not rendered by default.

149 changes: 83 additions & 66 deletions compiler/parser/parser.peg
Original file line number Diff line number Diff line change
@@ -105,9 +105,9 @@ Operation
}, nil
}
/ "from" __ "(" trunks:FromLeg+ __ ")" {
return &ast.From{
Kind: "From",
Trunks: sliceOf[ast.Seq](trunks),
return &ast.Parallel{
Kind: "Parallel",
Paths: sliceOf[ast.Seq](trunks),
Loc: loc(c),
}, nil
}
@@ -156,9 +156,7 @@ FromLeg
}

FromSource
= File
/ Get
/ Pool
= FromKeyWord _ spec:FromBody { return spec, nil }
/ PassOp

ExprGuard = __ ((!"=>" Comparator) / AdditiveOperator / MultiplicativeOperator / ":" / "(" / "[" / "~")
@@ -294,7 +292,7 @@ AggAssignment
Agg
= !FuncGuard op:AggName __ "(" __ expr:(OverExpr / Expr)? __ ")" !(__ ".") where:WhereClause? {
agg := &ast.Agg{
Kind:"Agg",
Kind: "Agg",
Name: op.(string),
Loc: loc(c),
}
@@ -607,46 +605,90 @@ DebugOp
}

FromOp
= File
/ Get
/ From

File
= "file" _ path:Path format:FormatArg? sortKeys:OrderArg? {
return &ast.File{
Kind: "File",
Path: path.(ast.Pattern),
Format: nullableName(format),
SortKeys: sliceOf[ast.SortExpr](sortKeys),
= FromKeyWord _ f:FromBody {
from := f.(*ast.From)
from.Loc = loc(c)
return from, nil
}

FromKeyWord
= "from"
/ DeprecatedFroms

DeprecatedFroms = "get" / "file" / "pool"

FromBody
= entity:FromEntity args:FromArgs {
from := &ast.From{
Kind: "From",
Entity: entity.(ast.FromEntity),
nwt marked this conversation as resolved.
Show resolved Hide resolved
Loc: loc(c),
}, nil
}
if args != nil {
from.Args = args.(ast.FromArgs)
}
return from, nil
}

From
= "from" _ spec:PoolSpec {
return &ast.Pool{
Kind: "Pool",
Spec: spec.(ast.PoolSpec),
FromEntity
= url:UnquotedURL {
return &ast.Name{
Kind: "Name",
Text: url.(string),
Loc: loc(c),
}, nil
}

Pool
= "pool" _ spec:PoolSpec {
return &ast.Pool{
Kind: "Pool",
Spec: spec.(ast.PoolSpec),
/ Regexp
/ Glob
/ "*" !ExprGuard { return &ast.Glob{Kind: "Glob", Pattern: "*", Loc: loc(c)}, nil }
/ e:Array {
return &ast.ExprEntity{
Kind: "ExprEntity",
Expr: e.(ast.Expr),
Loc: loc(c),
}, nil
}
/ Name
/ meta:PoolMeta {
return &ast.LakeMeta{
Kind: "LakeMeta",
Meta: nullableName(meta),
Loc: loc(c),
}, nil
}

Get
= "get" _ url:Path format:FormatArg? sortKeys:OrderArg? method:MethodArg? headers:HeadersArg? body:BodyArg? {
h := &ast.HTTP{
Kind: "HTTP",
URL: url.(ast.Pattern),
FromArgs
= commit:PoolCommit meta:PoolMeta? tap:TapArg {
return &ast.PoolArgs{
Kind: "PoolArgs",
Commit: nullableName(commit),
Meta: nullableName(meta),
Tap: tap.(bool),
nwt marked this conversation as resolved.
Show resolved Hide resolved
Loc: loc(c),
}, nil
}
/ meta:PoolMeta tap:TapArg {
return &ast.PoolArgs{
Kind: "PoolArgs",
Meta: nullableName(meta),
Tap: tap.(bool),
nwt marked this conversation as resolved.
Show resolved Hide resolved
Loc: loc(c),
}, nil
}
/ format:FormatArg !(_ ("method" / "header" / "body")) {
return &ast.FormatArg{
Kind: "FormatArg",
Format: nullableName(format),
Loc: loc(c),
}, nil
}
/ format:FormatArg? method:MethodArg? headers:HeadersArg? body:BodyArg? {
if format == nil && method == nil && headers == nil && body == nil {
return nil, nil
}
h := &ast.HTTPArgs{
Kind: "HTTPArgs",
Format: nullableName(format),
SortKeys: sliceOf[ast.SortExpr](sortKeys),
Method: nullableName(method),
Body: nullableName(body),
Loc: loc(c),
@@ -657,19 +699,17 @@ Get
return h, nil
}

FormatArg = _ "format" _ n:Name { return n, nil }

MethodArg = _ "method" _ n:Name { return n, nil }

HeadersArg = _ "headers" _ v:Record { return v, nil }

BodyArg = _ "body" _ n:Name { return n, nil }

Path
// XXX deprecate this
= [0-9a-zA-Z!@$%^&*_=<>,./?:[\]{}~+-]+ {
return &ast.Name{Kind: "Name", Text: string(c.text), Loc: loc(c)}, nil
}
/ Name
UnquotedURL = ("http://" / "https://") URLChar+ { return string(c.text), nil }

URLChar = [0-9a-zA-Z!@$%&_=,./?:[\]~+-]

//XXX this should be a timestamp
PoolAt
@@ -678,32 +718,12 @@ PoolAt
//XXX this should allow 0x bytes format
KSUID = ([0-9a-zA-Z])+ { return string(c.text), nil }

PoolSpec
= pool:PoolName commit:PoolCommit? meta:PoolMeta? tap:TapArg {
return ast.PoolSpec{
Pool: pool.(ast.Pattern),
Commit: nullableName(commit),
Meta: nullableName(meta),
Tap: tap.(bool),
Loc: loc(c),
}, nil
}
/ meta:PoolMeta {
return ast.PoolSpec{Meta: meta.(*ast.Name),Loc:loc(c)}, nil
}

PoolCommit
= "@" n:Name { return n, nil }

PoolMeta
= ":" n:Name { return n, nil }

PoolName
= Regexp
/ Glob
/ "*" !ExprGuard { return &ast.Glob{Kind: "Glob", Pattern: "*", Loc: loc(c)}, nil }
/ Name

OrderArg
= _ "order" _ exprs:SortExprs {
return exprs, nil
@@ -716,7 +736,7 @@ SortExprs

SortExpr
= e:Expr order:(_ o:OrderSpec { return o, nil })? {
s := ast.SortExpr{Kind: "SortExpr", Expr: e.(ast.Expr), Loc:loc(c)}
s := ast.SortExpr{Kind: "SortExpr", Expr: e.(ast.Expr), Loc:loc(c)}
if order != nil {
s.Order = order.(*ast.ID)
}
@@ -732,9 +752,6 @@ TapArg
= _ "tap" { return true, nil }
/ "" { return false, nil }

FormatArg
= _ "format" _ n:Name { return n, nil }

PassOp
= "pass" !(__ "(") &EOKW {
return &ast.Pass{Kind: "Pass", Loc: loc(c)}, nil
26 changes: 15 additions & 11 deletions compiler/parser/ztests/from.yaml
Original file line number Diff line number Diff line change
@@ -12,24 +12,28 @@ script: |
outputs:
- name: stdout
data: |
from (
pool a =>
search x and pool and b
fork (
=>
from a
| search x and pool and b
)
=== No spaces around parentheses.
from (
file a
fork (
=>
from a
)
from (
get "http://a"
fork (
=>
from "http://a"
)
from (
pool a
fork (
=>
from a
)
=== No space before vertical bar.
file a
from a
| search b
get "http://a"
from "http://a"
| search b
from a
| search b
6 changes: 5 additions & 1 deletion compiler/reader.go
Original file line number Diff line number Diff line change
@@ -21,7 +21,11 @@ func (i *anyCompiler) NewQuery(rctx *runtime.Context, seq ast.Seq, readers []zio
if len(readers) != 1 {
return nil, fmt.Errorf("NewQuery: Zed program expected %d readers", len(readers))
}
return CompileWithSortKey(rctx, seq, readers[0], order.SortKey{})
job, err := NewJob(rctx, seq, data.NewSource(nil, nil), nil)
if err != nil {
return nil, err
}
return optimizeAndBuild(job, readers)
}

// XXX currently used only by group-by test, need to deprecate
14 changes: 6 additions & 8 deletions compiler/semantic/analyzer.go
Original file line number Diff line number Diff line change
@@ -89,16 +89,14 @@ func AddDefaultSource(ctx context.Context, seq *dag.Seq, source *data.Source, he
if _, err := source.PoolID(ctx, head.Pool); err != nil {
return err
}
pool := &ast.Pool{
Kind: "Pool",
Spec: ast.PoolSpec{
Pool: &ast.Name{
Kind: "Name",
Text: "HEAD",
},
fromHead := &ast.From{
Kind: "From",
Entity: &ast.Name{
Kind: "Name",
Text: "HEAD",
},
}
ops := newAnalyzer(ctx, source, head).semPool(pool)
ops := newAnalyzer(ctx, source, head).semFrom(fromHead, nil)
seq.Prepend(ops[0])
return nil
}
485 changes: 260 additions & 225 deletions compiler/semantic/op.go

Large diffs are not rendered by default.

30 changes: 15 additions & 15 deletions compiler/ztests/const-source.yaml
Original file line number Diff line number Diff line change
@@ -2,14 +2,14 @@ script: |
export SUPER_DB_LAKE=test
super db init -q
super db create -q test
super db compile -dag -C 'const POOL = "test" from POOL' | sed -e "s/[a-zA-Z0-9]\{27\}/XXX/"
super db compile -dag -C 'const POOL = "test" from [POOL]' | sed -e "s/[a-zA-Z0-9]\{27\}/XXX/"
echo "==="
super db compile -dag -C 'const FILE = "A.zson" file FILE'
super compile -dag -C 'const FILE = "A.zson" from [FILE]'
echo "==="
super db compile -dag -C 'const URL = "http://brimdata.io" get URL'
! super db compile -dag -C 'const POOL = 3.14 from POOL'
! super db compile -dag -C 'const FILE = 127.0.0.1 file FILE'
! super db compile -dag -C 'const URL = true get URL'
super db compile -dag -C 'const URL = "http://brimdata.io" get [URL]'
! super db compile -dag -C 'const POOL = 3.14 from [POOL]'
! super db compile -dag -C 'const FILE = 127.0.0.1 file [FILE]'
! super db compile -dag -C 'const URL = true get [URL]'
outputs:
- name: stdout
@@ -36,12 +36,12 @@ outputs:
)
- name: stderr
data: |
POOL: string value required at line 1, column 24:
const POOL = 3.14 from POOL
~~~~
FILE: string value required at line 1, column 29:
const FILE = 127.0.0.1 file FILE
~~~~
URL: string value required at line 1, column 22:
const URL = true get URL
~~~
from expression requires a string but encountered 3.14 at line 1, column 24:
const POOL = 3.14 from [POOL]
~~~~~~
from expression requires a string but encountered 127.0.0.1 at line 1, column 29:
const FILE = 127.0.0.1 file [FILE]
~~~~~~
from expression requires a string but encountered true at line 1, column 22:
const URL = true get [URL]
~~~~~
11 changes: 0 additions & 11 deletions compiler/ztests/from-error.yaml
Original file line number Diff line number Diff line change
@@ -1,22 +1,15 @@
script: |
! super compile -C -dag 'from p'
export SUPER_DB_LAKE=test
super db init -q
! super db compile -C -dag 'from test'
echo === >&2
! super db compile -C -dag 'from test*'
echo === >&2
! super db compile -C -dag 'from /test/'
echo === >&2
super db create -q test
! super db compile -C -dag 'from (pool * => count())'
outputs:
- name: stderr
data: |
"from pool" cannot be used without a lake at line 1, column 1:
from p
~~~~~~
test: pool not found at line 1, column 6:
from test
~~~~
@@ -28,7 +21,3 @@ outputs:
test: pool matching regexp not found at line 1, column 6:
from /test/
~~~~~~
===
=> not allowed after pool pattern in 'from' operator at line 1, column 7:
from (pool * => count())
~~~~~~
4 changes: 2 additions & 2 deletions compiler/ztests/join-subquery.yaml
Original file line number Diff line number Diff line change
@@ -6,9 +6,9 @@ script: |
outputs:
- name: stdout
data: |
file a
from a
| join (
file b
from b
) on c
===
file a
2 changes: 1 addition & 1 deletion docs/language/statements.md
Original file line number Diff line number Diff line change
@@ -172,7 +172,7 @@ the data source of a [`from` operator](operators/from.md). For example, we
quote the pool name in our program `count-pool.spq`
```mdtest-input count-pool.spq
op CountPool(pool_name): (
from pool_name | count()
from [pool_name] | count()
)
CountPool("example")
6 changes: 3 additions & 3 deletions lake/ztests/create-ksuid-name.yaml
Original file line number Diff line number Diff line change
@@ -2,9 +2,9 @@
script: |
export SUPER_DB_LAKE=test
super db init -q
super db create 2WwyVrZdEITo5WkKu1YsJC4dMjU
super db use 2WwyVrZdEITo5WkKu1YsJC4dMjU
super db query 'from 2WwyVrZdEITo5WkKu1YsJC4dMjU'
super db create "2WwyVrZdEITo5WkKu1YsJC4dMjU"
super db use "2WwyVrZdEITo5WkKu1YsJC4dMjU"
super db query 'from "2WwyVrZdEITo5WkKu1YsJC4dMjU"'
outputs:
- name: stdout
6 changes: 3 additions & 3 deletions lake/ztests/query-error.yaml
Original file line number Diff line number Diff line change
@@ -10,12 +10,12 @@ outputs:
- name: stderr
data: |
query must include a 'from' operator
cannot scan from unknown HEAD at line 1, column 6:
cannot resolve unknown HEAD at line 1, column 6:
from HEAD
~~~~
unknown lake metadata type "unknownmeta" in from operator at line 1, column 1:
unknown lake metadata type "unknownmeta" in from operator at line 1, column 6:
from :unknownmeta
~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~
doesnotexist: pool not found at line 1, column 6:
from doesnotexist
~~~~~~~~~~~~
8 changes: 4 additions & 4 deletions lake/ztests/revert-revert.yaml
Original file line number Diff line number Diff line change
@@ -2,8 +2,8 @@ script: |
export SUPER_DB_LAKE=test
super db init -q
super db create -use -q test
a=$(super db load a.zson | head -1 | awk '{print $1}')
b=$(super db load b.zson | head -1 | awk '{print $1}')
a=$(super db load a.jsup | head -1 | awk '{print $1}')
b=$(super db load b.jsup | head -1 | awk '{print $1}')
super db query -z "from test | sort this"
r=$(super db revert $a | awk '{print $5}')
echo ===
@@ -13,10 +13,10 @@ script: |
super db query -z "sort this"
inputs:
- name: a.zson
- name: a.jsup
data: |
{a:1}
- name: b.zson
- name: b.jsup
data: |
{b:1}
2 changes: 1 addition & 1 deletion runtime/sam/op/ztests/user-from.yaml
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@ inputs:
data: |
const FILE = "A.zson"
op test(path): (
file path | sort a
file [path] | sort a
)
test(FILE)
- name: A.zson
4 changes: 2 additions & 2 deletions runtime/sam/op/ztests/user-join.yaml
Original file line number Diff line number Diff line change
@@ -11,8 +11,8 @@ inputs:
- name: test.spq
data: |
op test(lpool, rpool, lkey, rkey, ldest, rsrc): (
from lpool
| inner join ( from rpool ) on lkey = rkey ldest := rsrc
from [lpool]
| inner join ( from [rpool] ) on lkey = rkey ldest := rsrc
)
test("fruit", "people", flavor, likes, eater, name)
- name: fruit.zson
2 changes: 1 addition & 1 deletion service/ztests/create-ksuid-name.yaml
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ script: |
source service.sh
super db create 2WwyVrZdEITo5WkKu1YsJC4dMjU
super db use 2WwyVrZdEITo5WkKu1YsJC4dMjU
super db query 'from 2WwyVrZdEITo5WkKu1YsJC4dMjU'
super db query 'from "2WwyVrZdEITo5WkKu1YsJC4dMjU"'
inputs:
- name: service.sh
4 changes: 2 additions & 2 deletions service/ztests/curl-query-error.yaml
Original file line number Diff line number Diff line change
@@ -17,9 +17,9 @@ outputs:
code 400
{"type":"Error","kind":"invalid operation","error":"no pool name given"}
code 400
{"type":"Error","kind":"invalid operation","error":"pool name missing at line 1, column 1:\nfrom HEAD\n~~~~~~~~~","compilation_errors":[{"Msg":"pool name missing","Pos":0,"End":8}]}
{"type":"Error","kind":"invalid operation","error":"cannot resolve unknown HEAD at line 1, column 6:\nfrom HEAD\n ~~~~","compilation_errors":[{"Msg":"cannot resolve unknown HEAD","Pos":5,"End":8}]}
code 400
{"type":"Error","kind":"invalid operation","error":"unknown lake metadata type \"unknownmeta\" in from operator at line 1, column 1:\nfrom :unknownmeta\n~~~~~~~~~~~~~~~~~","compilation_errors":[{"Msg":"unknown lake metadata type \"unknownmeta\" in from operator","Pos":0,"End":16}]}
{"type":"Error","kind":"invalid operation","error":"unknown lake metadata type \"unknownmeta\" in from operator at line 1, column 6:\nfrom :unknownmeta\n ~~~~~~~~~~~~","compilation_errors":[{"Msg":"unknown lake metadata type \"unknownmeta\" in from operator","Pos":5,"End":16}]}
code 400
{"type":"Error","kind":"invalid operation","error":"doesnotexist: pool not found at line 1, column 6:\nfrom doesnotexist\n ~~~~~~~~~~~~","compilation_errors":[{"Msg":"doesnotexist: pool not found","Pos":5,"End":16}]}
code 400
4 changes: 2 additions & 2 deletions service/ztests/query-bad-commit.yaml
Original file line number Diff line number Diff line change
@@ -10,6 +10,6 @@ inputs:
outputs:
- name: stderr
data: |
"doesnotexist": branch not found at line 1, column 6:
"doesnotexist": branch not found at line 1, column 11:
from test@doesnotexist
~~~~
~~~~~~~~~~~~
6 changes: 3 additions & 3 deletions service/ztests/query-describe.yaml
Original file line number Diff line number Diff line change
@@ -14,9 +14,9 @@ inputs:
- name: service.sh
- name: multifrom.spq
data: |
from (
pool test1
pool test2
fork (
=> from test1
=> from test2
) | put foo := "bar"
- name: agg.spq
data: |
8 changes: 4 additions & 4 deletions service/ztests/query-error.yaml
Original file line number Diff line number Diff line change
@@ -13,12 +13,12 @@ outputs:
- name: stderr
data: |
status code 400: no pool name given
pool name missing at line 1, column 1:
cannot resolve unknown HEAD at line 1, column 6:
from HEAD
~~~~~~~~~
unknown lake metadata type "unknownmeta" in from operator at line 1, column 1:
~~~~
unknown lake metadata type "unknownmeta" in from operator at line 1, column 6:
from :unknownmeta
~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~
doesnotexist: pool not found at line 1, column 6:
from doesnotexist
~~~~~~~~~~~~
122 changes: 49 additions & 73 deletions zfmt/ast.go
Original file line number Diff line number Diff line change
@@ -419,38 +419,11 @@ func (c *canon) op(p ast.Op) {
c.write(")")
case *ast.From:
c.next()
c.open("from (")
for _, seq := range p.Trunks {
c.ret()
c.source(seq[0].(ast.Source))
if len(seq) > 1 {
c.write(" =>")
c.open()
c.head = true
c.seq(seq[1:])
c.close()
}
}
c.close()
c.ret()
c.flush()
c.write(")")
case *ast.Pool:
c.next()
c.open("")
c.write("from ")
c.pool(p)
c.close()
case *ast.File:
c.next()
c.open("")
c.file(p)
c.close()
case *ast.HTTP:
c.next()
c.open("")
c.http(p)
c.close()
c.fromEntity(p.Entity)
if p.Args != nil {
c.fromArgs(p.Args)
}
case *ast.Summarize:
c.next()
c.open("summarize")
@@ -640,6 +613,19 @@ func (c *canon) op(p ast.Op) {
}
}

func (c *canon) fromEntity(e ast.FromEntity) {
switch e := e.(type) {
case *ast.ExprEntity:
c.expr(e.Expr, "")
case *ast.Glob, *ast.Regexp:
c.pattern(e)
case *ast.Name:
c.write(zson.QuotedName(e.Text))
default:
panic(fmt.Sprintf("unknown from expression: %T", e))
}
}

func (c *canon) over(o *ast.Over) {
c.next()
c.write("over ")
@@ -680,33 +666,28 @@ func (c *canon) scope(s *ast.Scope, parens bool) {
}
}

func (c *canon) pool(p *ast.Pool) {
//XXX TBD name, from, to, id etc
s := pattern(p.Spec.Pool)
if p.Spec.Commit != nil {
s += "@" + p.Spec.Commit.Text
func (c *canon) poolArgs(args *ast.PoolArgs) {
s := ""
if args.Commit != nil {
s += "@" + args.Commit.Text
}
if p.Spec.Meta != nil {
s += ":" + p.Spec.Meta.Text
if args.Meta != nil {
s += ":" + args.Meta.Text
}
if p.Spec.Tap {
if args.Tap {
s += " tap"
}
c.write(s)
}

func pattern(p ast.Pattern) string {
func (c *canon) pattern(p ast.FromEntity) {
switch p := p.(type) {
case nil:
return ""
case *ast.Glob:
return p.Pattern
c.write(p.Pattern)
case *ast.Regexp:
return "/" + p.Pattern + "/"
case *ast.Name:
return zson.QuotedName(p.Text)
c.write("/" + p.Pattern + "/")
default:
return fmt.Sprintf("(unknown pattern type %T)", p)
panic(fmt.Sprintf("(unknown pattern type %T)", p))
}
}

@@ -782,42 +763,37 @@ func IsSearch(e ast.Expr) bool {
}
}

func (c *canon) http(p *ast.HTTP) {
//XXX TBD other stuff
c.write("get %s", pattern(p.URL))
if p.Format != nil {
c.write(" format %s", zson.QuotedName(p.Format.Text))
func (c *canon) httpArgs(args *ast.HTTPArgs) {
if args.Format != nil {
c.write(" format %s", args.Format.Text)
}
if p.Method != nil {
c.write(" method %s", zson.QuotedName(p.Method.Text))
if args.Method != nil {
c.write(" method %s", zson.QuotedName(args.Method.Text))
}
if p.Headers != nil {
if args.Headers != nil {
c.write(" headers ")
c.expr(p.Headers, "")
c.expr(args.Headers, "")
}
if p.Body != nil {
c.write(" body %s", zson.QuotedName(p.Body.Text))
if args.Body != nil {
c.write(" body %s", zson.QuotedName(args.Body.Text))
}
}

func (c *canon) file(p *ast.File) {
//XXX TBD other stuff
c.write("file %s", pattern(p.Path))
if p.Format != nil {
c.write(" format %s", zson.QuotedName(p.Format.Text))
func (c *canon) formatArg(arg *ast.FormatArg) {
if arg.Format != nil {
c.write(" format %s", arg.Format.Text)
}
}

func (c *canon) source(src ast.Source) {
switch src := src.(type) {
case *ast.Pool:
c.write("pool ")
c.pool(src)
case *ast.HTTP:
c.http(src)
case *ast.File:
c.file(src)
func (c *canon) fromArgs(args ast.FromArgs) {
switch args := args.(type) {
case *ast.PoolArgs:
c.poolArgs(args)
case *ast.HTTPArgs:
c.httpArgs(args)
case *ast.FormatArg:
c.formatArg(args)
default:
c.write("unknown source type: %T", src)
panic(fmt.Sprintf("unknown argument type in from operaetor: %T", args))
}
}
9 changes: 0 additions & 9 deletions zfmt/dag.go
Original file line number Diff line number Diff line change
@@ -511,15 +511,6 @@ func (c *canonDAG) op(p dag.Op) {
if p.Format != "" {
c.write(" format %s", p.Format)
}
if !p.SortKeys.IsNil() {
c.write(" order")
for i, s := range p.SortKeys {
if i > 0 {
c.write(",")
}
c.write(" %s %s", s.Key, s.Order)
}
}
if p.Filter != nil {
c.write(" filter (")
c.expr(p.Filter, "")
8 changes: 4 additions & 4 deletions zfmt/ztests/decls.yaml
Original file line number Diff line number Diff line change
@@ -13,9 +13,9 @@ inputs:
op stamp(assignee): ( yield {...this, assignee, ts: now()} )
op nop(foo): ( pass )
op joinTest(left_file, right_file, left_key, right_key, left_dest, right_source): (
file "left_file"
from [left_file]
| inner join (
file "right_file"
from [right_file]
) on left_key = right_key left_dest := right_source
)
joinTest("fruit.json", "people.json", flavor, likes, eater, name)
@@ -39,9 +39,9 @@ outputs:
pass
)
op joinTest(left_file, right_file, left_key, right_key, left_dest, right_source): (
file left_file
from [left_file]
| join (
file right_file
from [right_file]
) on left_key=right_key left_dest:=right_source
)
joinTest("fruit.json", "people.json", flavor, likes, eater, name)
64 changes: 38 additions & 26 deletions zfmt/ztests/from.yaml
Original file line number Diff line number Diff line change
@@ -24,46 +24,58 @@ script: |
outputs:
- name: stdout
data: |
file path
from path
===
file path format f
from path format f
===
get "http://host/path"
from "http://host/path"
===
get "http://host/path" format f
from "http://host/path" format f
===
from foo
===
from foo*
===
from /foo/
===
from (
file path
get "http://host/path"
pool name
fork (
=>
from path
=>
from "http://host/path"
=>
from name
)
===
from (
file path format f
get "http://host/path" format g
pool name
fork (
=>
from path format f
=>
from "http://host/path" format g
=>
from name
)
===
from (
file path =>
head
get "http://host/path" =>
head
pool name =>
head
fork (
=>
from path
| head
=>
from "http://host/path"
| head
=>
from name
| head
)
===
from (
file path format f =>
head
get "http://host/path" format g =>
head
pool name =>
head
fork (
=>
from path format f
| head
=>
from "http://host/path" format g
| head
=>
from name
| head
)
6 changes: 3 additions & 3 deletions zfmt/ztests/get.yaml
Original file line number Diff line number Diff line change
@@ -7,8 +7,8 @@ script: |
outputs:
- name: stdout
data: |
get "http://host/path"
from "http://host/path"
===
get "http://host/path" format f method m headers {a:["b"]} body b
from "http://host/path" format f method m headers {a:["b"]} body b
===
get "http://host/path" method "m|" body "b|"
from "http://host/path" method "m|" body "b|"
8 changes: 4 additions & 4 deletions zfmt/ztests/join.yaml
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
script: |
super compile -C "join (file test.zson) on x=x p:=a"
super compile -C "join (file test.jsup) on x=x p:=a"
echo ===
super compile -C -dag "join (file test.zson) on x=x p:=a"
super compile -C -dag "join (file test.jsup) on x=x p:=a"
outputs:
- name: stdout
data: |
join (
file "test.zson"
from "test.jsup"
) on x=x p:=a
===
reader
| fork (
=>
pass
=>
file test.zson
file test.jsup
)
| join on x=x p:=a
| output main