diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 1cc27fa01a08f..1508432d245c5 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -78,41 +78,41 @@ func (f *IPLineFilter) filterTy(line []byte, ty labels.MatchType) bool { type IPLabelFilter struct { ip *ipFilter - ty LabelFilterType + Ty LabelFilterType - // if used as label matcher, this holds the identifier label name. + // if used as Label matcher, this holds the identifier Label name. // e.g: (|remote_addr = ip("xxx")). Here labelName is `remote_addr` - label string + Label string // patError records if given pattern is invalid. patError error - // local copy of pattern to display it in errors, even though pattern matcher fails because of invalid pattern. - pattern string + // local copy of Pattern to display it in errors, even though Pattern matcher fails because of invalid Pattern. + Pattern string } // NewIPLabelFilter is used to construct ip filter as label filter for the given `label`. -func NewIPLabelFilter(pattern string, label string, ty LabelFilterType) *IPLabelFilter { +func NewIPLabelFilter(pattern, label string, ty LabelFilterType) *IPLabelFilter { ip, err := newIPFilter(pattern) return &IPLabelFilter{ ip: ip, - label: label, - ty: ty, + Label: label, + Ty: ty, patError: err, - pattern: pattern, + Pattern: pattern, } } // `Process` implements `Stage` interface func (f *IPLabelFilter) Process(_ int64, line []byte, lbs *LabelsBuilder) ([]byte, bool) { - return line, f.filterTy(line, f.ty, lbs) + return line, f.filterTy(line, f.Ty, lbs) } func (f *IPLabelFilter) isLabelFilterer() {} // `RequiredLabelNames` implements `Stage` interface func (f *IPLabelFilter) RequiredLabelNames() []string { - return []string{f.label} + return []string{f.Label} } // PatternError will be used `labelFilter.Stage()` method so that, if the given pattern is wrong @@ -126,7 +126,7 @@ func (f *IPLabelFilter) filterTy(_ []byte, ty LabelFilterType, lbs *LabelsBuilde // why `true`?. if there's an error only the string matchers can filter out. return true } - input, ok := lbs.Get(f.label) + input, ok := lbs.Get(f.Label) if !ok { // we have not found the label. return false @@ -148,11 +148,11 @@ func (f *IPLabelFilter) filterTy(_ []byte, ty LabelFilterType, lbs *LabelsBuilde // `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. func (f *IPLabelFilter) String() string { eq := "=" // LabelFilterEqual -> "==", we don't want in string representation of ip label filter. - if f.ty == LabelFilterNotEqual { + if f.Ty == LabelFilterNotEqual { eq = LabelFilterNotEqual.String() } - return fmt.Sprintf("%s%sip(%q)", f.label, eq, f.pattern) // label filter + return fmt.Sprintf("%s%sip(%q)", f.Label, eq, f.Pattern) // label filter } // ipFilter search for IP addresses of given `pattern` in the given `line`. diff --git a/pkg/logql/log/label_filter.go b/pkg/logql/log/label_filter.go index 2378902bce778..65f7a952302ea 100644 --- a/pkg/logql/log/label_filter.go +++ b/pkg/logql/log/label_filter.go @@ -67,7 +67,7 @@ type LabelFilterer interface { type BinaryLabelFilter struct { Left LabelFilterer Right LabelFilterer - and bool + And bool } // NewAndLabelFilter creates a new LabelFilterer from a and binary operation of two LabelFilterer. @@ -75,7 +75,7 @@ func NewAndLabelFilter(left LabelFilterer, right LabelFilterer) *BinaryLabelFilt return &BinaryLabelFilter{ Left: left, Right: right, - and: true, + And: true, } } @@ -89,11 +89,11 @@ func NewOrLabelFilter(left LabelFilterer, right LabelFilterer) *BinaryLabelFilte func (b *BinaryLabelFilter) Process(ts int64, line []byte, lbs *LabelsBuilder) ([]byte, bool) { line, lok := b.Left.Process(ts, line, lbs) - if !b.and && lok { + if !b.And && lok { return line, true } line, rok := b.Right.Process(ts, line, lbs) - if !b.and { + if !b.And { return line, lok || rok } return line, lok && rok @@ -112,7 +112,7 @@ func (b *BinaryLabelFilter) String() string { var sb strings.Builder sb.WriteString("( ") sb.WriteString(b.Left.String()) - if b.and { + if b.And { sb.WriteString(" , ") } else { sb.WriteString(" or ") @@ -219,7 +219,7 @@ func (d *BytesLabelFilter) String() string { return -1 } return r - }, humanize.IBytes(d.Value)) // TODO: discuss whether this should just be bytes, B, to be more accurate. + }, humanize.Bytes(d.Value)) // TODO: discuss whether this should just be bytes, B, to be more accurate. return fmt.Sprintf("%s%s%s", d.Name, d.Type, b) } @@ -364,7 +364,7 @@ func NewStringLabelFilter(m *labels.Matcher) LabelFilterer { return &NoopLabelFilter{m} } - return &lineFilterLabelFilter{ + return &LineFilterLabelFilter{ Matcher: m, filter: f, } @@ -380,14 +380,14 @@ func (s *StringLabelFilter) RequiredLabelNames() []string { return []string{s.Name} } -// lineFilterLabelFilter filters the desired label using an optimized line filter -type lineFilterLabelFilter struct { +// LineFilterLabelFilter filters the desired label using an optimized line filter +type LineFilterLabelFilter struct { *labels.Matcher filter Filterer } // overrides the matcher.String() function in case there is a regexpFilter -func (s *lineFilterLabelFilter) String() string { +func (s *LineFilterLabelFilter) String() string { if unwrappedFilter, ok := s.filter.(regexpFilter); ok { rStr := unwrappedFilter.String() str := fmt.Sprintf("%s%s`%s`", s.Matcher.Name, s.Matcher.Type, rStr) @@ -396,14 +396,14 @@ func (s *lineFilterLabelFilter) String() string { return s.Matcher.String() } -func (s *lineFilterLabelFilter) Process(_ int64, line []byte, lbs *LabelsBuilder) ([]byte, bool) { +func (s *LineFilterLabelFilter) Process(_ int64, line []byte, lbs *LabelsBuilder) ([]byte, bool) { v := labelValue(s.Name, lbs) return line, s.filter.Filter(unsafeGetBytes(v)) } -func (s *lineFilterLabelFilter) isLabelFilterer() {} +func (s *LineFilterLabelFilter) isLabelFilterer() {} -func (s *lineFilterLabelFilter) RequiredLabelNames() []string { +func (s *LineFilterLabelFilter) RequiredLabelNames() []string { return []string{s.Name} } diff --git a/pkg/logql/syntax/serialize.go b/pkg/logql/syntax/serialize.go index 6d14af9e27b7b..3dddcd2e90bbb 100644 --- a/pkg/logql/syntax/serialize.go +++ b/pkg/logql/syntax/serialize.go @@ -6,6 +6,7 @@ import ( "time" jsoniter "github.com/json-iterator/go" + "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/logql/log" ) @@ -318,7 +319,7 @@ func encodeUnwrap(s *jsoniter.Stream, u *UnwrapExpr) { if i > 0 { s.WriteMore() } - encodePostFilter(s, filter) + encodeLabelFilter(s, filter) } s.WriteArrayEnd() @@ -334,27 +335,42 @@ func decodeUnwrap(iter *jsoniter.Iterator) *UnwrapExpr { case "operation": e.Operation = iter.ReadString() case "post_filterers": - iter.Skip() + iter.ReadArrayCB(func(i *jsoniter.Iterator) bool { + e.PostFilters = append(e.PostFilters, decodeLabelFilter(i)) + return true + }) } } return e } -func encodePostFilter(s *jsoniter.Stream, filter log.LabelFilterer) { +func encodeLabelFilter(s *jsoniter.Stream, filter log.LabelFilterer) { switch concrete := filter.(type) { case *log.BinaryLabelFilter: + s.WriteObjectStart() + s.WriteObjectField("binary") + s.WriteObjectStart() s.WriteObjectField("left") - encodePostFilter(s, concrete.Left) + encodeLabelFilter(s, concrete.Left) s.WriteMore() s.WriteObjectField("right") - encodePostFilter(s, concrete.Right) + encodeLabelFilter(s, concrete.Right) + s.WriteObjectEnd() + + s.WriteMore() + s.WriteObjectField("and") + s.WriteBool(concrete.And) + s.WriteObjectEnd() case log.NoopLabelFilter: return case *log.BytesLabelFilter: + s.WriteObjectStart() + s.WriteObjectField("bytes") + s.WriteObjectStart() s.WriteObjectField("name") s.WriteString(concrete.Name) @@ -367,11 +383,220 @@ func encodePostFilter(s *jsoniter.Stream, filter log.LabelFilterer) { s.WriteObjectField("type") s.WriteInt(int(concrete.Type)) s.WriteObjectEnd() + + s.WriteObjectEnd() case *log.DurationLabelFilter: + s.WriteObjectStart() + s.WriteObjectField("duration") + + s.WriteObjectStart() + s.WriteObjectField("name") + s.WriteString(concrete.Name) + + s.WriteMore() + s.WriteObjectField("value") + s.WriteInt64(int64(concrete.Value)) + + s.WriteMore() + s.WriteObjectField("type") + s.WriteInt(int(concrete.Type)) + s.WriteObjectEnd() + + s.WriteObjectEnd() case *log.NumericLabelFilter: + s.WriteObjectStart() + s.WriteObjectField("numeric") + + s.WriteObjectStart() + s.WriteObjectField("name") + s.WriteString(concrete.Name) + + s.WriteMore() + s.WriteObjectField("value") + s.WriteFloat64(concrete.Value) + + s.WriteMore() + s.WriteObjectField("type") + s.WriteInt(int(concrete.Type)) + s.WriteObjectEnd() + + s.WriteObjectEnd() case *log.StringLabelFilter: - //case *log.lineFilterLabelFilter: + s.WriteObjectStart() + s.WriteObjectField("string") + + s.WriteObjectStart() + if concrete.Matcher != nil { + s.WriteObjectField("name") + s.WriteString(concrete.Name) + + s.WriteMore() + s.WriteObjectField("value") + s.WriteString(concrete.Value) + + s.WriteMore() + s.WriteObjectField("type") + s.WriteInt(int(concrete.Type)) + } + s.WriteObjectEnd() + + s.WriteObjectEnd() + case *log.LineFilterLabelFilter: + // Line filter label filter are encoded as string filters as + // well. See log.NewStringLabelFilter. + s.WriteObjectStart() + s.WriteObjectField("string") + + s.WriteObjectStart() + if concrete.Matcher != nil { + s.WriteObjectField("name") + s.WriteString(concrete.Name) + + s.WriteMore() + s.WriteObjectField("value") + s.WriteString(concrete.Value) + + s.WriteMore() + s.WriteObjectField("type") + s.WriteInt(int(concrete.Type)) + } + s.WriteObjectEnd() + + s.WriteObjectEnd() + case *log.IPLabelFilter: + s.WriteObjectStart() + s.WriteObjectField("ip") + + s.WriteObjectStart() + s.WriteObjectField("ty") + s.WriteInt(int(concrete.Ty)) + + s.WriteMore() + s.WriteObjectField("label") + s.WriteString(concrete.Label) + + s.WriteMore() + s.WriteObjectField("pattern") + s.WriteString(concrete.Pattern) + + s.WriteObjectEnd() + + s.WriteObjectEnd() + } +} + +func decodeLabelFilter(iter *jsoniter.Iterator) log.LabelFilterer { + for f := iter.ReadObject(); f != ""; f = iter.ReadObject() { + switch f { + case "binary": + var left, right log.LabelFilterer + var and bool + for k := iter.ReadObject(); k != ""; k = iter.ReadObject() { + switch k { + case "and": + and = iter.ReadBool() + case "left": + left = decodeLabelFilter(iter) + case "right": + right = decodeLabelFilter(iter) + } + } + + return &log.BinaryLabelFilter{ + And: and, + Left: left, + Right: right, + } + + case "bytes": + var name string + var b uint64 + var t log.LabelFilterType + for k := iter.ReadObject(); k != ""; k = iter.ReadObject() { + switch k { + case "name": + name = iter.ReadString() + case "value": + b = iter.ReadUint64() + case "type": + t = log.LabelFilterType(iter.ReadInt()) + } + } + return log.NewBytesLabelFilter(t, name, b) + case "duration": + var name string + var duration time.Duration + var t log.LabelFilterType + for k := iter.ReadObject(); k != ""; k = iter.ReadObject() { + switch k { + case "name": + name = iter.ReadString() + case "value": + duration = time.Duration(iter.ReadInt64()) + case "type": + t = log.LabelFilterType(iter.ReadInt()) + } + } + + return log.NewDurationLabelFilter(t, name, duration) + case "numeric": + var name string + var value float64 + var t log.LabelFilterType + for k := iter.ReadObject(); k != ""; k = iter.ReadObject() { + switch k { + case "name": + name = iter.ReadString() + case "value": + value = iter.ReadFloat64() + case "type": + t = log.LabelFilterType(iter.ReadInt()) + } + } + + return log.NewNumericLabelFilter(t, name, value) + case "string": + + var name string + var value string + var t labels.MatchType + for k := iter.ReadObject(); k != ""; k = iter.ReadObject() { + switch k { + case "name": + name = iter.ReadString() + case "value": + value = iter.ReadString() + case "type": + t = labels.MatchType(iter.ReadInt()) + } + } + + var matcher *labels.Matcher + if name != "" && value != "" { + matcher = labels.MustNewMatcher(t, name, value) + } + + return log.NewStringLabelFilter(matcher) + + case "ip": + var label string + var pattern string + var t log.LabelFilterType + for k := iter.ReadObject(); k != ""; k = iter.ReadObject() { + switch k { + case "pattern": + label = iter.ReadString() + case "label": + pattern = iter.ReadString() + case "ty": + t = log.LabelFilterType(iter.ReadInt()) + } + } + return log.NewIPLabelFilter(pattern, label, t) + } } + + return nil } func encodeLogSelector(s *jsoniter.Stream, e LogSelectorExpr) { diff --git a/pkg/logql/syntax/serialize_test.go b/pkg/logql/syntax/serialize_test.go index 9ae73b4c1070d..85e7f9a2cbfce 100644 --- a/pkg/logql/syntax/serialize_test.go +++ b/pkg/logql/syntax/serialize_test.go @@ -39,7 +39,7 @@ func TestJSONSerializationRoundTrip(t *testing.T) { query: `label_replace(vector(0.000000),"foo","bar","","")`, }, "filters with bytes": { - query: `{app="foo"} |= "bar" | json | ( status_code<500 or ( status_code>200 , size>=2.5KiB ) )`, + query: `{app="foo"} |= "bar" | json | ( status_code <500 or ( status_code>200 , size>=2.5KiB ) )`, }, "post filters": { query: `quantile_over_time(0.99998,{app="foo"} |= "bar" | json | latency >= 250ms or ( status_code < 500 and status_code > 200) @@ -62,7 +62,8 @@ func TestJSONSerializationRoundTrip(t *testing.T) { actual, err := DecodeJSON(buf.String()) require.NoError(t, err) - require.Equal(t, expr.Pretty(0), actual.Pretty(0)) + require.Equal(t, expr.String(), actual.String()) + //require.Equal(t, expr.Pretty(0), actual.Pretty(0)) }) } } diff --git a/vendor/github.com/json-iterator/go/iter_float.go b/vendor/github.com/json-iterator/go/iter_float.go index 8a3d8b6fb43c2..caf16feec89d9 100644 --- a/vendor/github.com/json-iterator/go/iter_float.go +++ b/vendor/github.com/json-iterator/go/iter_float.go @@ -66,7 +66,7 @@ func (iter *Iterator) ReadBigInt() (ret *big.Int) { return ret } -//ReadFloat32 read float32 +// ReadFloat32 read float32 func (iter *Iterator) ReadFloat32() (ret float32) { c := iter.nextToken() if c == '-' { diff --git a/vendor/github.com/json-iterator/go/iter_skip_strict.go b/vendor/github.com/json-iterator/go/iter_skip_strict.go index 6cf66d0438dbe..f1ad6591bb0c4 100644 --- a/vendor/github.com/json-iterator/go/iter_skip_strict.go +++ b/vendor/github.com/json-iterator/go/iter_skip_strict.go @@ -1,4 +1,5 @@ -//+build !jsoniter_sloppy +//go:build !jsoniter_sloppy +// +build !jsoniter_sloppy package jsoniter diff --git a/vendor/github.com/json-iterator/go/reflect_dynamic.go b/vendor/github.com/json-iterator/go/reflect_dynamic.go index 8b6bc8b433286..71a0fe273011f 100644 --- a/vendor/github.com/json-iterator/go/reflect_dynamic.go +++ b/vendor/github.com/json-iterator/go/reflect_dynamic.go @@ -1,9 +1,10 @@ package jsoniter import ( - "github.com/modern-go/reflect2" "reflect" "unsafe" + + "github.com/modern-go/reflect2" ) type dynamicEncoder struct { diff --git a/vendor/github.com/json-iterator/go/reflect_extension.go b/vendor/github.com/json-iterator/go/reflect_extension.go index 74a97bfe5abfb..a820f10cab787 100644 --- a/vendor/github.com/json-iterator/go/reflect_extension.go +++ b/vendor/github.com/json-iterator/go/reflect_extension.go @@ -2,12 +2,13 @@ package jsoniter import ( "fmt" - "github.com/modern-go/reflect2" "reflect" "sort" "strings" "unicode" "unsafe" + + "github.com/modern-go/reflect2" ) var typeDecoders = map[string]ValDecoder{} diff --git a/vendor/github.com/json-iterator/go/reflect_json_number.go b/vendor/github.com/json-iterator/go/reflect_json_number.go index 98d45c1ec2550..52e11bf3fb11c 100644 --- a/vendor/github.com/json-iterator/go/reflect_json_number.go +++ b/vendor/github.com/json-iterator/go/reflect_json_number.go @@ -2,9 +2,10 @@ package jsoniter import ( "encoding/json" - "github.com/modern-go/reflect2" "strconv" "unsafe" + + "github.com/modern-go/reflect2" ) type Number string diff --git a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go index eba434f2f16a3..521e38a89f302 100644 --- a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go +++ b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go @@ -2,8 +2,9 @@ package jsoniter import ( "encoding/json" - "github.com/modern-go/reflect2" "unsafe" + + "github.com/modern-go/reflect2" ) var jsonRawMessageType = reflect2.TypeOfPtr((*json.RawMessage)(nil)).Elem() diff --git a/vendor/github.com/json-iterator/go/reflect_map.go b/vendor/github.com/json-iterator/go/reflect_map.go index 5829671301353..696194bde7ef2 100644 --- a/vendor/github.com/json-iterator/go/reflect_map.go +++ b/vendor/github.com/json-iterator/go/reflect_map.go @@ -2,11 +2,12 @@ package jsoniter import ( "fmt" - "github.com/modern-go/reflect2" "io" "reflect" "sort" "unsafe" + + "github.com/modern-go/reflect2" ) func decoderOfMap(ctx *ctx, typ reflect2.Type) ValDecoder { diff --git a/vendor/github.com/json-iterator/go/reflect_optional.go b/vendor/github.com/json-iterator/go/reflect_optional.go index fa71f47489121..112c110a2932d 100644 --- a/vendor/github.com/json-iterator/go/reflect_optional.go +++ b/vendor/github.com/json-iterator/go/reflect_optional.go @@ -1,8 +1,9 @@ package jsoniter import ( - "github.com/modern-go/reflect2" "unsafe" + + "github.com/modern-go/reflect2" ) func decoderOfOptional(ctx *ctx, typ reflect2.Type) ValDecoder { diff --git a/vendor/github.com/json-iterator/go/reflect_slice.go b/vendor/github.com/json-iterator/go/reflect_slice.go index 9441d79df33b4..f363a7169b8c0 100644 --- a/vendor/github.com/json-iterator/go/reflect_slice.go +++ b/vendor/github.com/json-iterator/go/reflect_slice.go @@ -2,9 +2,10 @@ package jsoniter import ( "fmt" - "github.com/modern-go/reflect2" "io" "unsafe" + + "github.com/modern-go/reflect2" ) func decoderOfSlice(ctx *ctx, typ reflect2.Type) ValDecoder { diff --git a/vendor/github.com/json-iterator/go/reflect_struct_encoder.go b/vendor/github.com/json-iterator/go/reflect_struct_encoder.go index 152e3ef5a93c6..edf77bf597f2e 100644 --- a/vendor/github.com/json-iterator/go/reflect_struct_encoder.go +++ b/vendor/github.com/json-iterator/go/reflect_struct_encoder.go @@ -2,10 +2,11 @@ package jsoniter import ( "fmt" - "github.com/modern-go/reflect2" "io" "reflect" "unsafe" + + "github.com/modern-go/reflect2" ) func encoderOfStruct(ctx *ctx, typ reflect2.Type) ValEncoder {