From 67bb587a3dcbd4bb3807ca6ab27d0603ca105903 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 16:43:58 +0200 Subject: [PATCH 01/21] Start a new mango2 package --- x/mango2/doc.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 x/mango2/doc.go diff --git a/x/mango2/doc.go b/x/mango2/doc.go new file mode 100644 index 000000000..618365dff --- /dev/null +++ b/x/mango2/doc.go @@ -0,0 +1,17 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package mango2 provides support for parsing and executing Cloudant, CouchDB, +// and PouchDB-style [Mango queries] against JSON documents. +// +// [Mango queries]: https://docs.couchdb.org/en/stable/api/database/find.html#find-selectors +package mango2 From 41b2dc5122c780a754ae10305d76e0ba468dad12 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 17:16:05 +0200 Subject: [PATCH 02/21] Add basic types for Mango selector AST --- x/mango2/ast/constants.go | 51 ++++++++++++++++++++++ x/mango2/ast/selector.go | 92 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 x/mango2/ast/constants.go create mode 100644 x/mango2/ast/selector.go diff --git a/x/mango2/ast/constants.go b/x/mango2/ast/constants.go new file mode 100644 index 000000000..510d72896 --- /dev/null +++ b/x/mango2/ast/constants.go @@ -0,0 +1,51 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +package ast + +// Operator represents a Mango [operator]. +// +// [operator]: https://docs.couchdb.org/en/stable/api/database/find.html#explicit-operators +type Operator string + +// [Combination Operators] +// +// [Combination Operators]: https://docs.couchdb.org/en/stable/api/database/find.html#combination-operators +const ( + OpAnd = Operator("$and") + OpOr = Operator("$or") + OpNot = Operator("$not") + OpNor = Operator("$nor") + OpAll = Operator("$all") + OpElemMatch = Operator("$elemMatch") + OpAllMatch = Operator("$allMatch") + OpKeyMapMatch = Operator("$keyMapMatch") +) + +// [Condition Operators] +// +// [Condition Operators]: https://docs.couchdb.org/en/stable/api/database/find.html#condition-operators +const ( + OpLessThan = Operator("$lt") + OpLessThanOrEqual = Operator("$lte") + OpEqual = Operator("$eq") + OpNotEqual = Operator("$ne") + OpGreaterThan = Operator("$gt") + OpGreaterThanOrEqual = Operator("$gte") + OpExists = Operator("$exists") + OpType = Operator("$type") + OpIn = Operator("$in") + OpNotIn = Operator("$nin") + OpSize = Operator("$size") + OpMod = Operator("$mod") + OpRegex = Operator("$regex") +) diff --git a/x/mango2/ast/selector.go b/x/mango2/ast/selector.go new file mode 100644 index 000000000..c2d89b6db --- /dev/null +++ b/x/mango2/ast/selector.go @@ -0,0 +1,92 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// Package ast provides the abstract syntax tree for Mango selectors. +package ast + +// Selector represents a node in the Mango Selector. +type Selector interface { + Op() Operator + Value() interface{} +} + +type unarySelector struct { + op Operator + sel Selector +} + +var _ Selector = (*unarySelector)(nil) + +func (u *unarySelector) Op() Operator { + return u.op +} + +func (u *unarySelector) Value() interface{} { + return u.sel +} + +type combinationSelector struct { + op Operator + sel []Selector +} + +var _ Selector = (*combinationSelector)(nil) + +func (c *combinationSelector) Op() Operator { + return c.op +} + +func (c *combinationSelector) Value() interface{} { + return c.sel +} + +type conditionSelector struct { + op Operator + value interface{} +} + +var _ Selector = (*conditionSelector)(nil) + +func (e *conditionSelector) Op() Operator { + return e.op +} + +func (e *conditionSelector) Value() interface{} { + return e.value +} + +/* + + - $and []Selector + - $or []Selector + - $not Selector + - $nor []Selector + - $all []Selector + - $elemMatch Selector + - $allMatch Selector + - $keyMapMatch Selector + + - $lt Any JSON + - $lte Any JSON + - $eq Any JSON + - $ne Any JSON + - $gt Any JSON + - $gte Any JSON + - $exists Boolean + - $type String + - $in Array + - $nin Array + - $size Integer + - $mod Divisor and Remainder + - $regex String + +*/ From 0c416ad4f15135653b8c3a1e7fc130cb75bfcdef Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 17:37:21 +0200 Subject: [PATCH 03/21] Begin parsing selector queries into ASTs --- x/mango2/ast/ast.go | 63 +++++++++++++++++++++++++++ x/mango2/ast/ast_test.go | 94 ++++++++++++++++++++++++++++++++++++++++ x/mango2/ast/selector.go | 1 + 3 files changed, 158 insertions(+) create mode 100644 x/mango2/ast/ast.go create mode 100644 x/mango2/ast/ast_test.go diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go new file mode 100644 index 000000000..e045fc82d --- /dev/null +++ b/x/mango2/ast/ast.go @@ -0,0 +1,63 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +package ast + +import "encoding/json" + +// Parse parses s into a Mango Selector tree. +func Parse(input []byte) (Selector, error) { + var tmp map[string]json.RawMessage + if err := json.Unmarshal(input, &tmp); err != nil { + return nil, err + } + if len(tmp) == 0 { + return &combinationSelector{ + op: OpAnd, + sel: nil, + }, nil + } + if len(tmp) == 1 { + for k, v := range tmp { + op, value, err := opAndValue(v) + if err != nil { + return nil, err + } + return &conditionSelector{ + field: k, + op: op, + value: value, + }, nil + } + } + panic("not implemented") +} + +func opAndValue(input json.RawMessage) (Operator, interface{}, error) { + if input[0] != '{' { + var value interface{} + if err := json.Unmarshal(input, &value); err != nil { + return "", nil, err + } + return OpEqual, value, nil + } + var tmp map[string]interface{} + if err := json.Unmarshal(input, &tmp); err != nil { + return "", nil, err + } + if len(tmp) == 1 { + for k, v := range tmp { + return Operator(k), v, nil + } + } + return "", nil, nil +} diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go new file mode 100644 index 000000000..d88e9b1af --- /dev/null +++ b/x/mango2/ast/ast_test.go @@ -0,0 +1,94 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +package ast + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "gitlab.com/flimzy/testy" +) + +var cmpOpts = cmp.AllowUnexported(unarySelector{}, combinationSelector{}, conditionSelector{}) + +func TestParse(t *testing.T) { + type test struct { + input string + want Selector + wantErr string + } + + tests := testy.NewTable() + tests.Add("empty", test{ + input: "{}", + want: &combinationSelector{ + op: OpAnd, + sel: nil, + }, + }) + tests.Add("implicit equality", test{ + input: `{"foo": "bar"}`, + want: &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + }) + tests.Add("explicit equality", test{ + input: `{"foo": {"$eq": "bar"}}`, + want: &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + }) + + /* + TODO: + - explicit equality against an object + - $lt + - $lte + - $eq + - $ne + - $gt + - $gte + - $exists + - $type + - $in + - $nin + - $size + - $mod + - $regex + - implicit $and + - $and + - $or + - $not + - $nor + - $all + - $elemMatch + - $allMatch + - $keyMapMatch + + + */ + + tests.Run(t, func(t *testing.T, tt test) { + got, err := Parse([]byte(tt.input)) + if !testy.ErrorMatches(tt.wantErr, err) { + t.Fatalf("Unexpected error: %s", err) + } + if d := cmp.Diff(tt.want, got, cmpOpts); d != "" { + t.Errorf("Unexpected result (-want +got):\n%s", d) + } + }) +} diff --git a/x/mango2/ast/selector.go b/x/mango2/ast/selector.go index c2d89b6db..278551caa 100644 --- a/x/mango2/ast/selector.go +++ b/x/mango2/ast/selector.go @@ -50,6 +50,7 @@ func (c *combinationSelector) Value() interface{} { } type conditionSelector struct { + field string op Operator value interface{} } From fc8ef77f296f57f4d5c9c28701cc3e8bb7cddd1f Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 17:47:40 +0200 Subject: [PATCH 04/21] Test implicit equality against empty object --- x/mango2/ast/ast.go | 17 +++++++++++++---- x/mango2/ast/ast_test.go | 13 +++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index e045fc82d..d888700ff 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -12,7 +12,11 @@ package ast -import "encoding/json" +import ( + "encoding/json" + "errors" + "fmt" +) // Parse parses s into a Mango Selector tree. func Parse(input []byte) (Selector, error) { @@ -30,7 +34,7 @@ func Parse(input []byte) (Selector, error) { for k, v := range tmp { op, value, err := opAndValue(v) if err != nil { - return nil, err + return nil, fmt.Errorf("%s for operator %s", err, OpEqual) } return &conditionSelector{ field: k, @@ -54,10 +58,15 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { if err := json.Unmarshal(input, &tmp); err != nil { return "", nil, err } - if len(tmp) == 1 { + switch len(tmp) { + case 0: + return OpEqual, map[string]interface{}{}, nil + case 1: for k, v := range tmp { return Operator(k), v, nil } + default: + return "", nil, errors.New("too many keys in object") } - return "", nil, nil + panic("impossible") } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index d88e9b1af..c48eb1c38 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -52,9 +52,22 @@ func TestParse(t *testing.T) { value: "bar", }, }) + tests.Add("explicit equality with too many object keys", test{ + input: `{"foo": {"$eq": "bar", "$ne": "baz"}}`, + wantErr: "too many keys in object for operator $eq", + }) + tests.Add("implicit equality with empty object", test{ + input: `{"foo": {}}`, + want: &conditionSelector{ + field: "foo", + op: OpEqual, + value: map[string]interface{}{}, + }, + }) /* TODO: + - explicit equality against object with invalid operator -- error - explicit equality against an object - $lt - $lte From b536e92317f5b0e09e8b51ff4f690c9bcdba54ce Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 17:52:29 +0200 Subject: [PATCH 05/21] A few more equality corner cases --- x/mango2/ast/ast.go | 8 ++++++-- x/mango2/ast/ast_test.go | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index d888700ff..e9261d373 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -34,7 +34,7 @@ func Parse(input []byte) (Selector, error) { for k, v := range tmp { op, value, err := opAndValue(v) if err != nil { - return nil, fmt.Errorf("%s for operator %s", err, OpEqual) + return nil, err } return &conditionSelector{ field: k, @@ -63,7 +63,11 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return OpEqual, map[string]interface{}{}, nil case 1: for k, v := range tmp { - return Operator(k), v, nil + switch op := Operator(k); op { + case OpEqual: + return op, v, nil + } + return "", nil, fmt.Errorf("invalid operator %s", k) } default: return "", nil, errors.New("too many keys in object") diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index c48eb1c38..6e6b18258 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -54,7 +54,7 @@ func TestParse(t *testing.T) { }) tests.Add("explicit equality with too many object keys", test{ input: `{"foo": {"$eq": "bar", "$ne": "baz"}}`, - wantErr: "too many keys in object for operator $eq", + wantErr: "too many keys in object", }) tests.Add("implicit equality with empty object", test{ input: `{"foo": {}}`, @@ -64,11 +64,21 @@ func TestParse(t *testing.T) { value: map[string]interface{}{}, }, }) + tests.Add("explicit invalid comparison operator", test{ + input: `{"foo": {"$invalid": "bar"}}`, + wantErr: "invalid operator $invalid", + }) + tests.Add("explicit equiality against object", test{ + input: `{"foo": {"$eq": {"bar": "baz"}}}`, + want: &conditionSelector{ + field: "foo", + op: OpEqual, + value: map[string]interface{}{"bar": "baz"}, + }, + }) /* TODO: - - explicit equality against object with invalid operator -- error - - explicit equality against an object - $lt - $lte - $eq From 347504ae614cdf016d520e27b785fe3db3bc7791 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 17:57:31 +0200 Subject: [PATCH 06/21] Handle parsing for all equality operators --- x/mango2/ast/ast.go | 6 +++++- x/mango2/ast/ast_test.go | 46 ++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index e9261d373..f76cc2d57 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -46,6 +46,9 @@ func Parse(input []byte) (Selector, error) { panic("not implemented") } +// opAndValue is called when the input is an object in a context where a +// comparison operator is expected. It returns the operator and value, +// defaulting to [OpEqual] if no operator is specified. func opAndValue(input json.RawMessage) (Operator, interface{}, error) { if input[0] != '{' { var value interface{} @@ -64,7 +67,8 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { case 1: for k, v := range tmp { switch op := Operator(k); op { - case OpEqual: + case OpEqual, OpLessThan, OpLessThanOrEqual, OpNotEqual, + OpGreaterThan, OpGreaterThanOrEqual: return op, v, nil } return "", nil, fmt.Errorf("invalid operator %s", k) diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 6e6b18258..b2f0a8d54 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -76,15 +76,49 @@ func TestParse(t *testing.T) { value: map[string]interface{}{"bar": "baz"}, }, }) + tests.Add("less than", test{ + input: `{"foo": {"$lt": 42}}`, + want: &conditionSelector{ + field: "foo", + op: OpLessThan, + value: float64(42), + }, + }) + tests.Add("less than or equal", test{ + input: `{"foo": {"$lte": 42}}`, + want: &conditionSelector{ + field: "foo", + op: OpLessThanOrEqual, + value: float64(42), + }, + }) + tests.Add("not equal", test{ + input: `{"foo": {"$ne": 42}}`, + want: &conditionSelector{ + field: "foo", + op: OpNotEqual, + value: float64(42), + }, + }) + tests.Add("greater than", test{ + input: `{"foo": {"$gt": 42}}`, + want: &conditionSelector{ + field: "foo", + op: OpGreaterThan, + value: float64(42), + }, + }) + tests.Add("greater than or equal", test{ + input: `{"foo": {"$gte": 42}}`, + want: &conditionSelector{ + field: "foo", + op: OpGreaterThanOrEqual, + value: float64(42), + }, + }) /* TODO: - - $lt - - $lte - - $eq - - $ne - - $gt - - $gte - $exists - $type - $in From 9b07c4ecf62cdacb9ba8c82fbe03a863683bfe11 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 17:59:55 +0200 Subject: [PATCH 07/21] support for $exists --- x/mango2/ast/ast.go | 6 ++++++ x/mango2/ast/ast_test.go | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index f76cc2d57..579537782 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -70,6 +70,12 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { case OpEqual, OpLessThan, OpLessThanOrEqual, OpNotEqual, OpGreaterThan, OpGreaterThanOrEqual: return op, v, nil + case OpExists: + boolVal, ok := v.(bool) + if !ok { + return "", nil, fmt.Errorf("invalid value %v for $exists", v) + } + return OpExists, boolVal, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index b2f0a8d54..c45ebd59d 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -116,10 +116,21 @@ func TestParse(t *testing.T) { value: float64(42), }, }) + tests.Add("exists", test{ + input: `{"foo": {"$exists": true}}`, + want: &conditionSelector{ + field: "foo", + op: OpExists, + value: true, + }, + }) + tests.Add("exists with non-boolean", test{ + input: `{"foo": {"$exists": 42}}`, + wantErr: "invalid value 42 for $exists", + }) /* TODO: - - $exists - $type - $in - $nin From b33f911bdf5d4f49d02cd50fc9897da746752d0e Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 18:08:55 +0200 Subject: [PATCH 08/21] $in and $nin --- x/mango2/ast/ast.go | 26 +++++++++++++++++++------ x/mango2/ast/ast_test.go | 41 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 579537782..29727717f 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -57,7 +57,7 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { } return OpEqual, value, nil } - var tmp map[string]interface{} + var tmp map[string]json.RawMessage if err := json.Unmarshal(input, &tmp); err != nil { return "", nil, err } @@ -69,13 +69,27 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { switch op := Operator(k); op { case OpEqual, OpLessThan, OpLessThanOrEqual, OpNotEqual, OpGreaterThan, OpGreaterThanOrEqual: - return op, v, nil + var value interface{} + err := json.Unmarshal(v, &value) + return op, value, err case OpExists: - boolVal, ok := v.(bool) - if !ok { - return "", nil, fmt.Errorf("invalid value %v for $exists", v) + var value bool + if err := json.Unmarshal(v, &value); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + return OpExists, value, nil + case OpType: + var value string + if err := json.Unmarshal(v, &value); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + return OpType, value, nil + case OpIn, OpNotIn: + var value []interface{} + if err := json.Unmarshal(v, &value); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) } - return OpExists, boolVal, nil + return op, value, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index c45ebd59d..9951ace97 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -126,14 +126,47 @@ func TestParse(t *testing.T) { }) tests.Add("exists with non-boolean", test{ input: `{"foo": {"$exists": 42}}`, - wantErr: "invalid value 42 for $exists", + wantErr: "$exists: json: cannot unmarshal number into Go value of type bool", + }) + tests.Add("type", test{ + input: `{"foo": {"$type": "string"}}`, + want: &conditionSelector{ + field: "foo", + op: OpType, + value: "string", + }, + }) + tests.Add("type with non-string", test{ + input: `{"foo": {"$type": 42}}`, + wantErr: "$type: json: cannot unmarshal number into Go value of type string", + }) + tests.Add("in", test{ + input: `{"foo": {"$in": [1, 2, 3]}}`, + want: &conditionSelector{ + field: "foo", + op: OpIn, + value: []interface{}{float64(1), float64(2), float64(3)}, + }, + }) + tests.Add("in with non-array", test{ + input: `{"foo": {"$in": 42}}`, + wantErr: "$in: json: cannot unmarshal number into Go value of type []interface {}", + }) + tests.Add("not in", test{ + input: `{"foo": {"$nin": [1, 2, 3]}}`, + want: &conditionSelector{ + field: "foo", + op: OpNotIn, + value: []interface{}{float64(1), float64(2), float64(3)}, + }, + }) + tests.Add("not in with non-array", test{ + input: `{"foo": {"$nin": 42}}`, + wantErr: "$nin: json: cannot unmarshal number into Go value of type []interface {}", }) /* TODO: - - $type - - $in - - $nin - $size - $mod - $regex From e93c4731184aa0dba76ce5ecebdc4d59b7746da1 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 18:10:13 +0200 Subject: [PATCH 09/21] Add $size --- x/mango2/ast/ast.go | 6 ++++++ x/mango2/ast/ast_test.go | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 29727717f..ae16476ba 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -90,6 +90,12 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, fmt.Errorf("%s: %w", k, err) } return op, value, nil + case OpSize: + var value uint + if err := json.Unmarshal(v, &value); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + return OpSize, float64(value), nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 9951ace97..64e9507ec 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -164,10 +164,21 @@ func TestParse(t *testing.T) { input: `{"foo": {"$nin": 42}}`, wantErr: "$nin: json: cannot unmarshal number into Go value of type []interface {}", }) + tests.Add("size", test{ + input: `{"foo": {"$size": 42}}`, + want: &conditionSelector{ + field: "foo", + op: OpSize, + value: float64(42), + }, + }) + tests.Add("size with non-integer", test{ + input: `{"foo": {"$size": 42.5}}`, + wantErr: "$size: json: cannot unmarshal number 42.5 into Go value of type uint", + }) /* TODO: - - $size - $mod - $regex - implicit $and From 9e22d9ecb4791055d71d01496e9d11cc8f082d4c Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Mon, 15 Jul 2024 18:14:09 +0200 Subject: [PATCH 10/21] Add $mod --- x/mango2/ast/ast.go | 9 +++++++++ x/mango2/ast/ast_test.go | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index ae16476ba..fa381d472 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -96,6 +96,15 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, fmt.Errorf("%s: %w", k, err) } return OpSize, float64(value), nil + case OpMod: + var value [2]int + if err := json.Unmarshal(v, &value); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + if value[0] == 0 { + return "", nil, errors.New("$mod: divisor must be non-zero") + } + return OpMod, value, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 64e9507ec..f15220ab0 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -176,10 +176,25 @@ func TestParse(t *testing.T) { input: `{"foo": {"$size": 42.5}}`, wantErr: "$size: json: cannot unmarshal number 42.5 into Go value of type uint", }) + tests.Add("mod", test{ + input: `{"foo": {"$mod": [2, 1]}}`, + want: &conditionSelector{ + field: "foo", + op: OpMod, + value: [2]int{2, 1}, + }, + }) + tests.Add("mod with non-array", test{ + input: `{"foo": {"$mod": 42}}`, + wantErr: "$mod: json: cannot unmarshal number into Go value of type [2]int", + }) + tests.Add("mod with zero divisor", test{ + input: `{"foo": {"$mod": [0, 1]}}`, + wantErr: "$mod: divisor must be non-zero", + }) /* TODO: - - $mod - $regex - implicit $and - $and @@ -191,6 +206,7 @@ func TestParse(t *testing.T) { - $allMatch - $keyMapMatch + - $mod with non-integer values returns 404 (WTF) https://docs.couchdb.org/en/stable/api/database/find.html#condition-operators */ From 20886cae72207d0fbe2b1a5bcd4ab9bd907d1a22 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 09:07:45 +0200 Subject: [PATCH 11/21] Add $regex --- x/mango2/ast/ast.go | 11 +++++++++++ x/mango2/ast/ast_test.go | 24 +++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index fa381d472..7b1feafd8 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -16,6 +16,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" ) // Parse parses s into a Mango Selector tree. @@ -105,6 +106,16 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, errors.New("$mod: divisor must be non-zero") } return OpMod, value, nil + case OpRegex: + var pattern string + if err := json.Unmarshal(v, &pattern); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + re, err := regexp.Compile(pattern) + if err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + return OpRegex, re, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index f15220ab0..b1e1b690e 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -13,13 +13,16 @@ package ast import ( + "regexp" "testing" "github.com/google/go-cmp/cmp" "gitlab.com/flimzy/testy" ) -var cmpOpts = cmp.AllowUnexported(unarySelector{}, combinationSelector{}, conditionSelector{}) +var cmpOpts = []cmp.Option{ + cmp.AllowUnexported(unarySelector{}, combinationSelector{}, conditionSelector{}), +} func TestParse(t *testing.T) { type test struct { @@ -192,10 +195,25 @@ func TestParse(t *testing.T) { input: `{"foo": {"$mod": [0, 1]}}`, wantErr: "$mod: divisor must be non-zero", }) + tests.Add("regex", test{ + input: `{"foo": {"$regex": "^bar$"}}`, + want: &conditionSelector{ + field: "foo", + op: OpRegex, + value: regexp.MustCompile("^bar$"), + }, + }) + tests.Add("regexp non-string", test{ + input: `{"foo": {"$regex": 42}}`, + wantErr: "$regex: json: cannot unmarshal number into Go value of type string", + }) + tests.Add("regexp invalid", test{ + input: `{"foo": {"$regex": "["}}`, + wantErr: "$regex: error parsing regexp: missing closing ]: `[`", + }) /* TODO: - - $regex - implicit $and - $and - $or @@ -215,7 +233,7 @@ func TestParse(t *testing.T) { if !testy.ErrorMatches(tt.wantErr, err) { t.Fatalf("Unexpected error: %s", err) } - if d := cmp.Diff(tt.want, got, cmpOpts); d != "" { + if d := cmp.Diff(tt.want, got, cmpOpts...); d != "" { t.Errorf("Unexpected result (-want +got):\n%s", d) } }) From 45036a834d1ea71ab5c8a17bc2565cd0bd8f2529 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 10:49:14 +0200 Subject: [PATCH 12/21] Explicit $and --- x/mango2/ast/ast.go | 44 ++++++++++++++++++++--- x/mango2/ast/ast_test.go | 76 +++++++++++++++++++++++++++++++++++++-- x/mango2/ast/selector.go | 78 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 8 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 7b1feafd8..abcd831c5 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -17,6 +17,7 @@ import ( "errors" "fmt" "regexp" + "sort" ) // Parse parses s into a Mango Selector tree. @@ -26,25 +27,58 @@ func Parse(input []byte) (Selector, error) { return nil, err } if len(tmp) == 0 { + // Empty object is an implicit $and return &combinationSelector{ op: OpAnd, sel: nil, }, nil } - if len(tmp) == 1 { - for k, v := range tmp { + sels := make([]Selector, 0, len(tmp)) + for k, v := range tmp { + switch op := Operator(k); op { + case OpAnd: + var sel []json.RawMessage + if err := json.Unmarshal(v, &sel); err != nil { + return nil, err + } + subsels := make([]Selector, 0, len(sel)) + for _, s := range sel { + sel, err := Parse(s) + if err != nil { + return nil, err + } + subsels = append(subsels, sel) + } + + sels = append(sels, &combinationSelector{ + op: OpAnd, + sel: subsels, + }) + default: op, value, err := opAndValue(v) if err != nil { return nil, err } - return &conditionSelector{ + sels = append(sels, &conditionSelector{ field: k, op: op, value: value, - }, nil + }) } } - panic("not implemented") + if len(sels) == 1 { + return sels[0], nil + } + + // Sort the selectors to ensure deterministic output. + sort.Slice(sels, func(i, j int) bool { + return cmpSelectors(sels[i], sels[j]) < 0 + }) + + return &combinationSelector{ + op: OpAnd, + sel: sels, + }, nil } // opAndValue is called when the input is an object in a context where a diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index b1e1b690e..11065d40a 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -211,11 +211,78 @@ func TestParse(t *testing.T) { input: `{"foo": {"$regex": "["}}`, wantErr: "$regex: error parsing regexp: missing closing ]: `[`", }) + tests.Add("implicit $and", test{ + input: `{"foo":"bar","baz":"qux"}`, + want: &combinationSelector{ + op: OpAnd, + sel: []Selector{ + &conditionSelector{ + field: "baz", + op: OpEqual, + value: "qux", + }, + &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + }, + }, + }) + tests.Add("explicit $and", test{ + input: `{"$and":[{"foo":"bar"},{"baz":"qux"}]}`, + want: &combinationSelector{ + op: OpAnd, + sel: []Selector{ + &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + &conditionSelector{ + field: "baz", + op: OpEqual, + value: "qux", + }, + }, + }, + }) + tests.Add("nested implicit and explicit $and", test{ + input: `{"$and":[{"foo":"bar"},{"baz":"qux"}, {"quux":"corge","grault":"garply"}]}`, + want: &combinationSelector{ + op: OpAnd, + sel: []Selector{ + &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + &conditionSelector{ + field: "baz", + op: OpEqual, + value: "qux", + }, + &combinationSelector{ + op: OpAnd, + sel: []Selector{ + &conditionSelector{ + field: "grault", + op: OpEqual, + value: "garply", + }, + &conditionSelector{ + field: "quux", + op: OpEqual, + value: "corge", + }, + }, + }, + }, + }, + }) /* TODO: - - implicit $and - - $and - $or - $not - $nor @@ -233,7 +300,10 @@ func TestParse(t *testing.T) { if !testy.ErrorMatches(tt.wantErr, err) { t.Fatalf("Unexpected error: %s", err) } - if d := cmp.Diff(tt.want, got, cmpOpts...); d != "" { + if err != nil { + return + } + if d := cmp.Diff(tt.want.String(), got.String(), cmpOpts...); d != "" { t.Errorf("Unexpected result (-want +got):\n%s", d) } }) diff --git a/x/mango2/ast/selector.go b/x/mango2/ast/selector.go index 278551caa..6bbd0a19f 100644 --- a/x/mango2/ast/selector.go +++ b/x/mango2/ast/selector.go @@ -13,10 +13,16 @@ // Package ast provides the abstract syntax tree for Mango selectors. package ast +import ( + "fmt" + "strings" +) + // Selector represents a node in the Mango Selector. type Selector interface { Op() Operator Value() interface{} + String() string } type unarySelector struct { @@ -34,6 +40,10 @@ func (u *unarySelector) Value() interface{} { return u.sel } +func (u *unarySelector) String() string { + return fmt.Sprintf("%s %s", u.op, u.sel) +} + type combinationSelector struct { op Operator sel []Selector @@ -49,6 +59,20 @@ func (c *combinationSelector) Value() interface{} { return c.sel } +func (c *combinationSelector) String() string { + var sb strings.Builder + sb.WriteString(string(c.op)) + sb.WriteString(" [") + for i, sel := range c.sel { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%v", sel)) + } + sb.WriteString("]") + return sb.String() +} + type conditionSelector struct { field string op Operator @@ -65,6 +89,10 @@ func (e *conditionSelector) Value() interface{} { return e.value } +func (e *conditionSelector) String() string { + return fmt.Sprintf("%s %s %v", e.field, e.op, e.value) +} + /* - $and []Selector @@ -91,3 +119,53 @@ func (e *conditionSelector) Value() interface{} { - $regex String */ + +// cmpValues compares two arbitrary values by converting them to strings. +func cmpValues(a, b interface{}) int { + return strings.Compare(fmt.Sprintf("%v", a), fmt.Sprintf("%v", b)) +} + +// cmpSelectors compares two selectors, for ordering. +func cmpSelectors(a, b Selector) int { + // Naively sort operators alphabetically. + if c := strings.Compare(string(a.Op()), string(b.Op())); c != 0 { + return c + } + switch t := a.(type) { + case *unarySelector: + u := b.(*unarySelector) + return cmpSelectors(t.sel, u.sel) + case *combinationSelector: + u := b.(*combinationSelector) + for i := 0; i < len(t.sel) && i < len(u.sel); i++ { + if c := cmpSelectors(t.sel[i], u.sel[i]); c != 0 { + return c + } + } + return len(t.sel) - len(u.sel) + case *conditionSelector: + u := b.(*conditionSelector) + if c := strings.Compare(t.field, u.field); c != 0 { + return c + } + switch t.op { + case OpIn, OpNotIn: + for i := 0; i < len(t.value.([]interface{})) && i < len(u.value.([]interface{})); i++ { + if c := cmpValues(t.value.([]interface{})[i], u.value.([]interface{})[i]); c != 0 { + return c + } + } + return len(t.value.([]interface{})) - len(u.value.([]interface{})) + case OpMod: + tm := t.value.([2]int) + um := u.value.([2]int) + if tm[0] != um[0] { + return tm[0] - um[0] + } + return tm[1] - um[1] + default: + return cmpValues(t.value, u.value) + } + } + return 0 +} From 7d6dd60a40ef89f0a244bb5c8442864975ab066b Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 10:53:22 +0200 Subject: [PATCH 13/21] Support $or --- x/mango2/ast/ast.go | 7 +++++-- x/mango2/ast/ast_test.go | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index abcd831c5..b86fb1067 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -36,7 +36,7 @@ func Parse(input []byte) (Selector, error) { sels := make([]Selector, 0, len(tmp)) for k, v := range tmp { switch op := Operator(k); op { - case OpAnd: + case OpAnd, OpOr: var sel []json.RawMessage if err := json.Unmarshal(v, &sel); err != nil { return nil, err @@ -51,10 +51,13 @@ func Parse(input []byte) (Selector, error) { } sels = append(sels, &combinationSelector{ - op: OpAnd, + op: op, sel: subsels, }) default: + if op[0] == '$' { + return nil, fmt.Errorf("unknown operator %s", op) + } op, value, err := opAndValue(v) if err != nil { return nil, err diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 11065d40a..8d1e5bab8 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -280,10 +280,27 @@ func TestParse(t *testing.T) { }, }, }) + tests.Add("$or", test{ + input: `{"$or":[{"foo":"bar"},{"baz":"qux"}]}`, + want: &combinationSelector{ + op: OpOr, + sel: []Selector{ + &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + &conditionSelector{ + field: "baz", + op: OpEqual, + value: "qux", + }, + }, + }, + }) /* TODO: - - $or - $not - $nor - $all From 6f8b300533e28c18253b120b7548b2b14c4f37d0 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 10:53:46 +0200 Subject: [PATCH 14/21] Reject invalid operators --- x/mango2/ast/ast_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 8d1e5bab8..8a620ff1a 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -298,6 +298,10 @@ func TestParse(t *testing.T) { }, }, }) + tests.Add("invalid operator", test{ + input: `{"$invalid": "bar"}`, + wantErr: "unknown operator $invalid", + }) /* TODO: From f957105c741190e0593995796a8fdbf04f2ca71a Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 10:55:24 +0200 Subject: [PATCH 15/21] Add $not --- x/mango2/ast/ast.go | 9 +++++++++ x/mango2/ast/ast_test.go | 12 +++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index b86fb1067..dc5645792 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -54,6 +54,15 @@ func Parse(input []byte) (Selector, error) { op: op, sel: subsels, }) + case OpNot: + sel, err := Parse(v) + if err != nil { + return nil, err + } + sels = append(sels, &unarySelector{ + op: op, + sel: sel, + }) default: if op[0] == '$' { return nil, fmt.Errorf("unknown operator %s", op) diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 8a620ff1a..de7ee3dc0 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -302,10 +302,20 @@ func TestParse(t *testing.T) { input: `{"$invalid": "bar"}`, wantErr: "unknown operator $invalid", }) + tests.Add("$not", test{ + input: `{"$not": {"foo":"bar"}}`, + want: &unarySelector{ + op: OpNot, + sel: &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + }, + }) /* TODO: - - $not - $nor - $all - $elemMatch From a9fdc4bf3e4e02a948dfba49a1f5ccdfe1577c22 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 10:58:13 +0200 Subject: [PATCH 16/21] Tests for some error cases --- x/mango2/ast/ast.go | 6 +++--- x/mango2/ast/ast_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index dc5645792..4d458dcf5 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -39,13 +39,13 @@ func Parse(input []byte) (Selector, error) { case OpAnd, OpOr: var sel []json.RawMessage if err := json.Unmarshal(v, &sel); err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", k, err) } subsels := make([]Selector, 0, len(sel)) for _, s := range sel { sel, err := Parse(s) if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", k, err) } subsels = append(subsels, sel) } @@ -57,7 +57,7 @@ func Parse(input []byte) (Selector, error) { case OpNot: sel, err := Parse(v) if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", k, err) } sels = append(sels, &unarySelector{ op: op, diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index de7ee3dc0..7bf2c0322 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -313,6 +313,18 @@ func TestParse(t *testing.T) { }, }, }) + tests.Add("$not with invalid selector", test{ + input: `{"$not": []}`, + wantErr: "$not: json: cannot unmarshal array into Go value of type map[string]json.RawMessage", + }) + tests.Add("$and with invalid selector array", test{ + input: `{"$and": {}}`, + wantErr: "$and: json: cannot unmarshal object into Go value of type []json.RawMessage", + }) + tests.Add("$and with invalid selector", test{ + input: `{"$and": [42]}`, + wantErr: "$and: json: cannot unmarshal number into Go value of type map[string]json.RawMessag", + }) /* TODO: From e5c64bbbe6377f617c07dec0ac82c024374076d5 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 10:59:26 +0200 Subject: [PATCH 17/21] Add $nor --- x/mango2/ast/ast.go | 2 +- x/mango2/ast/ast_test.go | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 4d458dcf5..7dd43839e 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -36,7 +36,7 @@ func Parse(input []byte) (Selector, error) { sels := make([]Selector, 0, len(tmp)) for k, v := range tmp { switch op := Operator(k); op { - case OpAnd, OpOr: + case OpAnd, OpOr, OpNor: var sel []json.RawMessage if err := json.Unmarshal(v, &sel); err != nil { return nil, fmt.Errorf("%s: %w", k, err) diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 7bf2c0322..8c4baec21 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -323,12 +323,29 @@ func TestParse(t *testing.T) { }) tests.Add("$and with invalid selector", test{ input: `{"$and": [42]}`, - wantErr: "$and: json: cannot unmarshal number into Go value of type map[string]json.RawMessag", + wantErr: "$and: json: cannot unmarshal number into Go value of type map[string]json.RawMessage", + }) + tests.Add("$nor", test{ + input: `{"$nor":[{"foo":"bar"},{"baz":"qux"}]}`, + want: &combinationSelector{ + op: OpNor, + sel: []Selector{ + &conditionSelector{ + field: "foo", + op: OpEqual, + value: "bar", + }, + &conditionSelector{ + field: "baz", + op: OpEqual, + value: "qux", + }, + }, + }, }) /* TODO: - - $nor - $all - $elemMatch - $allMatch From 7434c6b06a709486f9f0ab0ee545d8860d8e5dd8 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 11:06:30 +0200 Subject: [PATCH 18/21] Add $all --- x/mango2/ast/ast.go | 6 ++++++ x/mango2/ast/ast_test.go | 13 ++++++++++++- x/mango2/ast/constants.go | 2 +- x/mango2/ast/selector.go | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 7dd43839e..1e694c03d 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -162,6 +162,12 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, fmt.Errorf("%s: %w", k, err) } return OpRegex, re, nil + case OpAll: + var value []interface{} + if err := json.Unmarshal(v, &value); err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + return OpAll, value, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 8c4baec21..2d656d7a4 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -343,10 +343,21 @@ func TestParse(t *testing.T) { }, }, }) + tests.Add("$all", test{ + input: `{"foo": {"$all": ["bar", "baz"]}}`, + want: &conditionSelector{ + field: "foo", + op: OpAll, + value: []interface{}{"bar", "baz"}, + }, + }) + tests.Add("$all with non-array", test{ + input: `{"foo": {"$all": "bar"}}`, + wantErr: "$all: json: cannot unmarshal string into Go value of type []interface {}", + }) /* TODO: - - $all - $elemMatch - $allMatch - $keyMapMatch diff --git a/x/mango2/ast/constants.go b/x/mango2/ast/constants.go index 510d72896..a51e4468c 100644 --- a/x/mango2/ast/constants.go +++ b/x/mango2/ast/constants.go @@ -25,7 +25,6 @@ const ( OpOr = Operator("$or") OpNot = Operator("$not") OpNor = Operator("$nor") - OpAll = Operator("$all") OpElemMatch = Operator("$elemMatch") OpAllMatch = Operator("$allMatch") OpKeyMapMatch = Operator("$keyMapMatch") @@ -48,4 +47,5 @@ const ( OpSize = Operator("$size") OpMod = Operator("$mod") OpRegex = Operator("$regex") + OpAll = Operator("$all") ) diff --git a/x/mango2/ast/selector.go b/x/mango2/ast/selector.go index 6bbd0a19f..98ee8a927 100644 --- a/x/mango2/ast/selector.go +++ b/x/mango2/ast/selector.go @@ -99,7 +99,6 @@ func (e *conditionSelector) String() string { - $or []Selector - $not Selector - $nor []Selector - - $all []Selector - $elemMatch Selector - $allMatch Selector - $keyMapMatch Selector @@ -117,6 +116,7 @@ func (e *conditionSelector) String() string { - $size Integer - $mod Divisor and Remainder - $regex String + - $all Array */ From 3f30205b7664f6a52bf46a01d09af7986d90550d Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 11:33:22 +0200 Subject: [PATCH 19/21] Add $elemMatch --- x/mango2/ast/ast.go | 40 +++++- x/mango2/ast/ast_test.go | 253 ++++++++++++++++++++++++-------------- x/mango2/ast/constants.go | 2 +- x/mango2/ast/selector.go | 48 +++++++- 4 files changed, 243 insertions(+), 100 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 1e694c03d..7f06413c0 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -63,19 +63,43 @@ func Parse(input []byte) (Selector, error) { op: op, sel: sel, }) - default: - if op[0] == '$' { - return nil, fmt.Errorf("unknown operator %s", op) - } + case OpEqual, OpLessThan, OpLessThanOrEqual, OpNotEqual, + OpGreaterThan, OpGreaterThanOrEqual: op, value, err := opAndValue(v) if err != nil { return nil, err } sels = append(sels, &conditionSelector{ - field: k, op: op, value: value, }) + default: + if op[0] == '$' { + return nil, fmt.Errorf("unknown operator %s", op) + } + op, value, err := opAndValue(v) + if err != nil { + return nil, err + } + + switch op { + case OpElemMatch: + sels = append(sels, &fieldSelector{ + field: k, + cond: &elementSelector{ + op: op, + cond: value.(*conditionSelector), + }, + }) + default: + sels = append(sels, &fieldSelector{ + field: k, + cond: &conditionSelector{ + op: op, + value: value, + }, + }) + } } } if len(sels) == 1 { @@ -168,6 +192,12 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, fmt.Errorf("%s: %w", k, err) } return OpAll, value, nil + case OpElemMatch: + sel, err := Parse(v) + if err != nil { + return "", nil, fmt.Errorf("%s: %w", k, err) + } + return OpElemMatch, sel, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 2d656d7a4..126a388cd 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -41,18 +41,22 @@ func TestParse(t *testing.T) { }) tests.Add("implicit equality", test{ input: `{"foo": "bar"}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, }) tests.Add("explicit equality", test{ input: `{"foo": {"$eq": "bar"}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, }) tests.Add("explicit equality with too many object keys", test{ @@ -61,10 +65,12 @@ func TestParse(t *testing.T) { }) tests.Add("implicit equality with empty object", test{ input: `{"foo": {}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpEqual, - value: map[string]interface{}{}, + cond: &conditionSelector{ + op: OpEqual, + value: map[string]interface{}{}, + }, }, }) tests.Add("explicit invalid comparison operator", test{ @@ -73,58 +79,72 @@ func TestParse(t *testing.T) { }) tests.Add("explicit equiality against object", test{ input: `{"foo": {"$eq": {"bar": "baz"}}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpEqual, - value: map[string]interface{}{"bar": "baz"}, + cond: &conditionSelector{ + op: OpEqual, + value: map[string]interface{}{"bar": "baz"}, + }, }, }) tests.Add("less than", test{ input: `{"foo": {"$lt": 42}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpLessThan, - value: float64(42), + cond: &conditionSelector{ + op: OpLessThan, + value: float64(42), + }, }, }) tests.Add("less than or equal", test{ input: `{"foo": {"$lte": 42}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpLessThanOrEqual, - value: float64(42), + cond: &conditionSelector{ + op: OpLessThanOrEqual, + value: float64(42), + }, }, }) tests.Add("not equal", test{ input: `{"foo": {"$ne": 42}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpNotEqual, - value: float64(42), + cond: &conditionSelector{ + op: OpNotEqual, + value: float64(42), + }, }, }) tests.Add("greater than", test{ input: `{"foo": {"$gt": 42}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpGreaterThan, - value: float64(42), + cond: &conditionSelector{ + op: OpGreaterThan, + value: float64(42), + }, }, }) tests.Add("greater than or equal", test{ input: `{"foo": {"$gte": 42}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpGreaterThanOrEqual, - value: float64(42), + cond: &conditionSelector{ + op: OpGreaterThanOrEqual, + value: float64(42), + }, }, }) tests.Add("exists", test{ input: `{"foo": {"$exists": true}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpExists, - value: true, + cond: &conditionSelector{ + op: OpExists, + value: true, + }, }, }) tests.Add("exists with non-boolean", test{ @@ -133,10 +153,12 @@ func TestParse(t *testing.T) { }) tests.Add("type", test{ input: `{"foo": {"$type": "string"}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpType, - value: "string", + cond: &conditionSelector{ + op: OpType, + value: "string", + }, }, }) tests.Add("type with non-string", test{ @@ -145,10 +167,12 @@ func TestParse(t *testing.T) { }) tests.Add("in", test{ input: `{"foo": {"$in": [1, 2, 3]}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpIn, - value: []interface{}{float64(1), float64(2), float64(3)}, + cond: &conditionSelector{ + op: OpIn, + value: []interface{}{float64(1), float64(2), float64(3)}, + }, }, }) tests.Add("in with non-array", test{ @@ -157,10 +181,12 @@ func TestParse(t *testing.T) { }) tests.Add("not in", test{ input: `{"foo": {"$nin": [1, 2, 3]}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpNotIn, - value: []interface{}{float64(1), float64(2), float64(3)}, + cond: &conditionSelector{ + op: OpNotIn, + value: []interface{}{float64(1), float64(2), float64(3)}, + }, }, }) tests.Add("not in with non-array", test{ @@ -169,10 +195,12 @@ func TestParse(t *testing.T) { }) tests.Add("size", test{ input: `{"foo": {"$size": 42}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpSize, - value: float64(42), + cond: &conditionSelector{ + op: OpSize, + value: float64(42), + }, }, }) tests.Add("size with non-integer", test{ @@ -181,10 +209,12 @@ func TestParse(t *testing.T) { }) tests.Add("mod", test{ input: `{"foo": {"$mod": [2, 1]}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpMod, - value: [2]int{2, 1}, + cond: &conditionSelector{ + op: OpMod, + value: [2]int{2, 1}, + }, }, }) tests.Add("mod with non-array", test{ @@ -197,10 +227,12 @@ func TestParse(t *testing.T) { }) tests.Add("regex", test{ input: `{"foo": {"$regex": "^bar$"}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpRegex, - value: regexp.MustCompile("^bar$"), + cond: &conditionSelector{ + op: OpRegex, + value: regexp.MustCompile("^bar$"), + }, }, }) tests.Add("regexp non-string", test{ @@ -216,15 +248,19 @@ func TestParse(t *testing.T) { want: &combinationSelector{ op: OpAnd, sel: []Selector{ - &conditionSelector{ + &fieldSelector{ field: "baz", - op: OpEqual, - value: "qux", + cond: &conditionSelector{ + op: OpEqual, + value: "qux", + }, }, - &conditionSelector{ + &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, }, }, @@ -234,15 +270,19 @@ func TestParse(t *testing.T) { want: &combinationSelector{ op: OpAnd, sel: []Selector{ - &conditionSelector{ + &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, - &conditionSelector{ + &fieldSelector{ field: "baz", - op: OpEqual, - value: "qux", + cond: &conditionSelector{ + op: OpEqual, + value: "qux", + }, }, }, }, @@ -252,28 +292,36 @@ func TestParse(t *testing.T) { want: &combinationSelector{ op: OpAnd, sel: []Selector{ - &conditionSelector{ + &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, - &conditionSelector{ + &fieldSelector{ field: "baz", - op: OpEqual, - value: "qux", + cond: &conditionSelector{ + op: OpEqual, + value: "qux", + }, }, &combinationSelector{ op: OpAnd, sel: []Selector{ - &conditionSelector{ + &fieldSelector{ field: "grault", - op: OpEqual, - value: "garply", + cond: &conditionSelector{ + op: OpEqual, + value: "garply", + }, }, - &conditionSelector{ + &fieldSelector{ field: "quux", - op: OpEqual, - value: "corge", + cond: &conditionSelector{ + op: OpEqual, + value: "corge", + }, }, }, }, @@ -285,15 +333,19 @@ func TestParse(t *testing.T) { want: &combinationSelector{ op: OpOr, sel: []Selector{ - &conditionSelector{ + &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, - &conditionSelector{ + &fieldSelector{ field: "baz", - op: OpEqual, - value: "qux", + cond: &conditionSelector{ + op: OpEqual, + value: "qux", + }, }, }, }, @@ -306,10 +358,12 @@ func TestParse(t *testing.T) { input: `{"$not": {"foo":"bar"}}`, want: &unarySelector{ op: OpNot, - sel: &conditionSelector{ + sel: &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, }, }) @@ -330,31 +384,50 @@ func TestParse(t *testing.T) { want: &combinationSelector{ op: OpNor, sel: []Selector{ - &conditionSelector{ + &fieldSelector{ field: "foo", - op: OpEqual, - value: "bar", + cond: &conditionSelector{ + op: OpEqual, + value: "bar", + }, }, - &conditionSelector{ + &fieldSelector{ field: "baz", - op: OpEqual, - value: "qux", + cond: &conditionSelector{ + op: OpEqual, + value: "qux", + }, }, }, }, }) tests.Add("$all", test{ input: `{"foo": {"$all": ["bar", "baz"]}}`, - want: &conditionSelector{ + want: &fieldSelector{ field: "foo", - op: OpAll, - value: []interface{}{"bar", "baz"}, + cond: &conditionSelector{ + op: OpAll, + value: []interface{}{"bar", "baz"}, + }, }, }) tests.Add("$all with non-array", test{ input: `{"foo": {"$all": "bar"}}`, wantErr: "$all: json: cannot unmarshal string into Go value of type []interface {}", }) + tests.Add("$elemMatch", test{ + input: `{"genre": {"$elemMatch": {"$eq": "Horror"}}}`, + want: &fieldSelector{ + field: "genre", + cond: &elementSelector{ + op: OpElemMatch, + cond: &conditionSelector{ + op: OpEqual, + value: "Horror", + }, + }, + }, + }) /* TODO: diff --git a/x/mango2/ast/constants.go b/x/mango2/ast/constants.go index a51e4468c..c6c53b0ef 100644 --- a/x/mango2/ast/constants.go +++ b/x/mango2/ast/constants.go @@ -25,7 +25,6 @@ const ( OpOr = Operator("$or") OpNot = Operator("$not") OpNor = Operator("$nor") - OpElemMatch = Operator("$elemMatch") OpAllMatch = Operator("$allMatch") OpKeyMapMatch = Operator("$keyMapMatch") ) @@ -48,4 +47,5 @@ const ( OpMod = Operator("$mod") OpRegex = Operator("$regex") OpAll = Operator("$all") + OpElemMatch = Operator("$elemMatch") ) diff --git a/x/mango2/ast/selector.go b/x/mango2/ast/selector.go index 98ee8a927..81ee1f4df 100644 --- a/x/mango2/ast/selector.go +++ b/x/mango2/ast/selector.go @@ -73,8 +73,26 @@ func (c *combinationSelector) String() string { return sb.String() } -type conditionSelector struct { +type fieldSelector struct { field string + cond Selector +} + +var _ Selector = (*fieldSelector)(nil) + +func (f *fieldSelector) Op() Operator { + return f.cond.Op() +} + +func (f *fieldSelector) Value() interface{} { + return f.cond.Value() +} + +func (f *fieldSelector) String() string { + return fmt.Sprintf("%s %s", f.field, f.cond.String()) +} + +type conditionSelector struct { op Operator value interface{} } @@ -90,7 +108,26 @@ func (e *conditionSelector) Value() interface{} { } func (e *conditionSelector) String() string { - return fmt.Sprintf("%s %s %v", e.field, e.op, e.value) + return fmt.Sprintf("%s %v", e.op, e.value) +} + +type elementSelector struct { + op Operator + cond *conditionSelector +} + +var _ Selector = (*elementSelector)(nil) + +func (e *elementSelector) Op() Operator { + return e.op +} + +func (e *elementSelector) Value() interface{} { + return e.cond +} + +func (e *elementSelector) String() string { + return fmt.Sprintf("%s {%s}", e.op, e.cond) } /* @@ -143,11 +180,14 @@ func cmpSelectors(a, b Selector) int { } } return len(t.sel) - len(u.sel) - case *conditionSelector: - u := b.(*conditionSelector) + case *fieldSelector: + u := b.(*fieldSelector) if c := strings.Compare(t.field, u.field); c != 0 { return c } + return cmpSelectors(t.cond, u.cond) + case *conditionSelector: + u := b.(*conditionSelector) switch t.op { case OpIn, OpNotIn: for i := 0; i < len(t.value.([]interface{})) && i < len(u.value.([]interface{})); i++ { From e8fc259c89fe49f5c4b21542f45c3aa9c7a6b24d Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 11:35:32 +0200 Subject: [PATCH 20/21] Add $allMatch --- x/mango2/ast/ast.go | 6 +++--- x/mango2/ast/ast_test.go | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 7f06413c0..608fcb3ce 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -83,7 +83,7 @@ func Parse(input []byte) (Selector, error) { } switch op { - case OpElemMatch: + case OpElemMatch, OpAllMatch: sels = append(sels, &fieldSelector{ field: k, cond: &elementSelector{ @@ -192,12 +192,12 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, fmt.Errorf("%s: %w", k, err) } return OpAll, value, nil - case OpElemMatch: + case OpElemMatch, OpAllMatch: sel, err := Parse(v) if err != nil { return "", nil, fmt.Errorf("%s: %w", k, err) } - return OpElemMatch, sel, nil + return op, sel, nil } return "", nil, fmt.Errorf("invalid operator %s", k) } diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 126a388cd..8f55b4620 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -428,11 +428,22 @@ func TestParse(t *testing.T) { }, }, }) + tests.Add("$allMatch", test{ + input: `{"genre": {"$allMatch": {"$eq": "Horror"}}}`, + want: &fieldSelector{ + field: "genre", + cond: &elementSelector{ + op: OpAllMatch, + cond: &conditionSelector{ + op: OpEqual, + value: "Horror", + }, + }, + }, + }) /* TODO: - - $elemMatch - - $allMatch - $keyMapMatch - $mod with non-integer values returns 404 (WTF) https://docs.couchdb.org/en/stable/api/database/find.html#condition-operators From d7db50ab9b77a83a975d627129f2544e67687932 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Tue, 16 Jul 2024 11:37:16 +0200 Subject: [PATCH 21/21] Add $keyMapMatch --- x/mango2/ast/ast.go | 4 ++-- x/mango2/ast/ast_test.go | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/x/mango2/ast/ast.go b/x/mango2/ast/ast.go index 608fcb3ce..e2ae7f6fc 100644 --- a/x/mango2/ast/ast.go +++ b/x/mango2/ast/ast.go @@ -83,7 +83,7 @@ func Parse(input []byte) (Selector, error) { } switch op { - case OpElemMatch, OpAllMatch: + case OpElemMatch, OpAllMatch, OpKeyMapMatch: sels = append(sels, &fieldSelector{ field: k, cond: &elementSelector{ @@ -192,7 +192,7 @@ func opAndValue(input json.RawMessage) (Operator, interface{}, error) { return "", nil, fmt.Errorf("%s: %w", k, err) } return OpAll, value, nil - case OpElemMatch, OpAllMatch: + case OpElemMatch, OpAllMatch, OpKeyMapMatch: sel, err := Parse(v) if err != nil { return "", nil, fmt.Errorf("%s: %w", k, err) diff --git a/x/mango2/ast/ast_test.go b/x/mango2/ast/ast_test.go index 8f55b4620..112ffd4f8 100644 --- a/x/mango2/ast/ast_test.go +++ b/x/mango2/ast/ast_test.go @@ -441,11 +441,26 @@ func TestParse(t *testing.T) { }, }, }) + tests.Add("$keyMapMatch", test{ + input: `{"cameras": {"$keyMapMatch": {"$eq": "secondary"}}}`, + want: &fieldSelector{ + field: "cameras", + cond: &elementSelector{ + op: OpKeyMapMatch, + cond: &conditionSelector{ + op: OpEqual, + value: "secondary", + }, + }, + }, + }) + tests.Add("element selector with invalid selector", test{ + input: `{"cameras": {"$keyMapMatch": 42}}`, + wantErr: "$keyMapMatch: json: cannot unmarshal number into Go value of type map[string]json.RawMessage", + }) /* TODO: - - $keyMapMatch - - $mod with non-integer values returns 404 (WTF) https://docs.couchdb.org/en/stable/api/database/find.html#condition-operators */