From 5b97fcfd93ae0aa341284b6557697495c445c196 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 22 Nov 2023 08:29:02 +0100 Subject: [PATCH] Send query plan to querier. (#11246) **What this PR does / why we need it**: Following https://github.com/grafana/loki/pull/11123 and in order to enable https://github.com/grafana/loki/pull/10417 the query frontend should send the serialized LogQL AST instead of the query string to the queriers. This enables the frontend to change the AST and inject expressions that are not expressible in LogQL. **Checklist** - [ ] Reviewed the [`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md) guide (**required**) - [ ] Documentation added - [x] Tests updated - [ ] `CHANGELOG.md` updated - [ ] If the change is worth mentioning in the release notes, add `add-to-release-notes` label - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md` - [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213) - [ ] If the change is deprecating or removing a configuration option, update the `deprecated-config.yaml` and `deleted-config.yaml` files respectively in the `tools/deprecated-config-checker` directory. [Example PR](https://github.com/grafana/loki/pull/10840/commits/0d4416a4b03739583349934b96f272fb4f685d15) --------- Signed-off-by: Callum Styan Co-authored-by: Callum Styan --- pkg/logcli/client/file.go | 10 +- pkg/logcli/query/query.go | 22 +- pkg/logcli/query/query_test.go | 5 +- pkg/logproto/indexgateway.pb.go | 50 ++- pkg/logproto/indexgateway.proto | 1 - pkg/logproto/sketch.pb.go | 78 ++-- pkg/logproto/sketch.proto | 1 - pkg/logql/blocker.go | 4 +- pkg/logql/blocker_test.go | 13 +- pkg/logql/downstream.go | 46 ++- pkg/logql/downstream_test.go | 25 +- pkg/logql/engine.go | 27 +- pkg/logql/engine_test.go | 160 +++----- pkg/logql/evaluator.go | 69 +++- pkg/logql/evaluator_test.go | 4 +- pkg/logql/explain_test.go | 10 +- pkg/logql/metrics.go | 12 +- pkg/logql/metrics_test.go | 49 ++- pkg/logql/rangemapper.go | 8 +- pkg/logql/rangemapper_test.go | 24 +- pkg/logql/shardmapper.go | 7 +- pkg/logql/shardmapper_test.go | 12 +- pkg/logql/syntax/parser.go | 8 + pkg/logql/test_utils.go | 12 +- pkg/querier/plan/plan.go | 101 +++++ pkg/querier/plan/plan_test.go | 26 ++ pkg/querier/queryrange/codec.go | 66 +++- pkg/querier/queryrange/codec_test.go | 19 +- pkg/querier/queryrange/downstreamer.go | 21 +- pkg/querier/queryrange/downstreamer_test.go | 58 ++- pkg/querier/queryrange/marshal.go | 24 ++ pkg/querier/queryrange/queryrange.pb.go | 345 ++++++++++++------ pkg/querier/queryrange/queryrange.proto | 2 + pkg/querier/queryrange/querysharding.go | 14 +- pkg/querier/queryrange/querysharding_test.go | 40 +- pkg/querier/queryrange/roundtrip_test.go | 20 +- pkg/querier/queryrange/split_by_interval.go | 2 + pkg/querier/queryrange/split_by_range.go | 16 +- pkg/querier/queryrange/split_by_range_test.go | 26 +- pkg/querier/queryrange/stats.go | 6 +- pkg/querier/queryrange/stats_test.go | 12 +- pkg/querier/worker/util_test.go | 2 +- pkg/ruler/evaluator_local.go | 5 +- 43 files changed, 932 insertions(+), 530 deletions(-) create mode 100644 pkg/querier/plan/plan.go create mode 100644 pkg/querier/plan/plan_test.go diff --git a/pkg/logcli/client/file.go b/pkg/logcli/client/file.go index 45681c36c2c8f..82274ef79fb8d 100644 --- a/pkg/logcli/client/file.go +++ b/pkg/logcli/client/file.go @@ -69,7 +69,7 @@ func (f *FileClient) Query(q string, limit int, t time.Time, direction logproto. ctx = user.InjectOrgID(ctx, f.orgID) - params := logql.NewLiteralParams( + params, err := logql.NewLiteralParams( q, t, t, 0, @@ -78,6 +78,9 @@ func (f *FileClient) Query(q string, limit int, t time.Time, direction logproto. uint32(limit), nil, ) + if err != nil { + return nil, fmt.Errorf("failed to parse query: %w", err) + } query := f.engine.Query(params) @@ -106,7 +109,7 @@ func (f *FileClient) QueryRange(queryStr string, limit int, start, end time.Time ctx = user.InjectOrgID(ctx, f.orgID) - params := logql.NewLiteralParams( + params, err := logql.NewLiteralParams( queryStr, start, end, @@ -116,6 +119,9 @@ func (f *FileClient) QueryRange(queryStr string, limit int, start, end time.Time uint32(limit), nil, ) + if err != nil { + return nil, err + } query := f.engine.Query(params) diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go index 6a71f0979abcf..fc5be5f393cb2 100644 --- a/pkg/logcli/query/query.go +++ b/pkg/logcli/query/query.go @@ -451,7 +451,7 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string var query logql.Query if q.isInstant() { - query = eng.Query(logql.NewLiteralParams( + params, err := logql.NewLiteralParams( q.QueryString, q.Start, q.Start, @@ -460,9 +460,14 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string q.resultsDirection(), uint32(q.Limit), nil, - )) + ) + if err != nil { + return err + } + + query = eng.Query(params) } else { - query = eng.Query(logql.NewLiteralParams( + params, err := logql.NewLiteralParams( q.QueryString, q.Start, q.End, @@ -471,7 +476,16 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string q.resultsDirection(), uint32(q.Limit), nil, - )) + ) + if err != nil { + return err + } + + query = eng.Query(params) + } + + if err != nil { + return err } // execute the query diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go index 72886fb84668d..1b4c18f5265e0 100644 --- a/pkg/logcli/query/query_test.go +++ b/pkg/logcli/query/query_test.go @@ -425,7 +425,10 @@ func (t *testQueryClient) Query(_ string, _ int, _ time.Time, _ logproto.Directi func (t *testQueryClient) QueryRange(queryStr string, limit int, from, through time.Time, direction logproto.Direction, step, interval time.Duration, _ bool) (*loghttp.QueryResponse, error) { ctx := user.InjectOrgID(context.Background(), "fake") - params := logql.NewLiteralParams(queryStr, from, through, step, interval, direction, uint32(limit), nil) + params, err := logql.NewLiteralParams(queryStr, from, through, step, interval, direction, uint32(limit), nil) + if err != nil { + return nil, err + } v, err := t.engine.Query(params).Exec(ctx) if err != nil { diff --git a/pkg/logproto/indexgateway.pb.go b/pkg/logproto/indexgateway.pb.go index e8b569ea07323..86b2665e86b17 100644 --- a/pkg/logproto/indexgateway.pb.go +++ b/pkg/logproto/indexgateway.pb.go @@ -6,7 +6,6 @@ package logproto import ( context "context" fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -28,31 +27,30 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("pkg/logproto/indexgateway.proto", fileDescriptor_d27585148d0a52c8) } var fileDescriptor_d27585148d0a52c8 = []byte{ - // 372 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xc1, 0x4a, 0xfb, 0x30, - 0x1c, 0xc7, 0x1b, 0xf8, 0xf3, 0x47, 0xa3, 0x78, 0x08, 0xc2, 0x46, 0xa7, 0x11, 0xc4, 0x83, 0x5e, - 0x56, 0xd1, 0x17, 0x10, 0x85, 0x95, 0xc1, 0x14, 0x9c, 0xb0, 0xc3, 0x0e, 0x62, 0x3a, 0x7f, 0xeb, - 0xca, 0xba, 0xa6, 0xb6, 0x29, 0xba, 0x9b, 0x8f, 0xe0, 0x63, 0xf8, 0x10, 0x3e, 0x80, 0xc7, 0x1d, - 0x77, 0x74, 0xdd, 0xc5, 0xe3, 0x1e, 0x41, 0x9a, 0xd0, 0x2d, 0x9b, 0x1d, 0x78, 0x6a, 0xfa, 0xf9, - 0x7e, 0xf3, 0xf9, 0xd1, 0xa4, 0xf8, 0x20, 0xec, 0xbb, 0x96, 0xcf, 0xdd, 0x30, 0xe2, 0x82, 0x5b, - 0x5e, 0xf0, 0x08, 0x2f, 0x2e, 0x13, 0xf0, 0xcc, 0x86, 0x55, 0x89, 0xc8, 0x8e, 0xce, 0x42, 0xc7, - 0xdc, 0x75, 0xb9, 0xcb, 0x55, 0x3b, 0x5b, 0xa9, 0x96, 0x59, 0x59, 0xd2, 0xe4, 0x0b, 0x15, 0x9e, - 0x7d, 0xfc, 0xc3, 0xdb, 0xf5, 0xcc, 0x62, 0x2b, 0x0b, 0xa9, 0x63, 0x7c, 0x9b, 0x40, 0x34, 0x94, - 0x90, 0x54, 0xaa, 0xf3, 0xfe, 0x82, 0x36, 0xe1, 0x29, 0x81, 0x58, 0x98, 0x7b, 0xc5, 0x61, 0x1c, - 0xf2, 0x20, 0x86, 0x53, 0x44, 0x1a, 0x78, 0xcb, 0x06, 0x71, 0xd5, 0x4b, 0x82, 0x7e, 0x13, 0xba, - 0x44, 0xab, 0x6b, 0x38, 0x97, 0xed, 0xaf, 0x49, 0x95, 0xed, 0xd0, 0x20, 0x35, 0xbc, 0x69, 0x83, - 0xb8, 0x83, 0xc8, 0x83, 0x98, 0x98, 0x4b, 0x6d, 0x05, 0x73, 0x53, 0xa5, 0x30, 0x9b, 0x7b, 0xee, - 0x71, 0xa9, 0xc1, 0x1c, 0xf0, 0x6f, 0xd8, 0x00, 0xe2, 0x1a, 0x8f, 0xae, 0x41, 0x44, 0x5e, 0x27, - 0x7b, 0x23, 0xc7, 0x8b, 0x9d, 0x6b, 0x2a, 0xf9, 0x8c, 0xd2, 0x4a, 0x53, 0xf3, 0x3f, 0xe0, 0xb2, - 0x44, 0x2d, 0xe6, 0x27, 0xab, 0x03, 0x4e, 0x56, 0xb6, 0x15, 0x74, 0xfe, 0x30, 0xc1, 0xc6, 0x1b, - 0xd9, 0x87, 0x09, 0x26, 0x62, 0xfd, 0x82, 0xe4, 0xf1, 0x4b, 0x5a, 0x70, 0x41, 0x7a, 0x38, 0x17, - 0x5d, 0xc8, 0x23, 0x6d, 0x71, 0x3f, 0x19, 0x00, 0xd1, 0x06, 0x2a, 0x92, 0x5b, 0xca, 0xbf, 0x83, - 0xdc, 0x70, 0xd9, 0x1e, 0x4d, 0xa8, 0x31, 0x9e, 0x50, 0x63, 0x36, 0xa1, 0xe8, 0x35, 0xa5, 0xe8, - 0x3d, 0xa5, 0xe8, 0x33, 0xa5, 0x68, 0x94, 0x52, 0xf4, 0x95, 0x52, 0xf4, 0x9d, 0x52, 0x63, 0x96, - 0x52, 0xf4, 0x36, 0xa5, 0xc6, 0x68, 0x4a, 0x8d, 0xf1, 0x94, 0x1a, 0xed, 0x23, 0xd7, 0x13, 0xbd, - 0xc4, 0xa9, 0x76, 0xf8, 0xc0, 0x72, 0x23, 0xd6, 0x65, 0x01, 0xb3, 0x7c, 0xde, 0xf7, 0x2c, 0xfd, - 0x4f, 0x75, 0xfe, 0xcb, 0xc7, 0xf9, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x79, 0xe4, 0x24, 0x34, - 0x07, 0x03, 0x00, 0x00, + // 361 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xb1, 0x4e, 0xc2, 0x40, + 0x18, 0xc7, 0xef, 0x12, 0x63, 0xf4, 0x34, 0x0e, 0xb7, 0x40, 0x40, 0xcf, 0xc4, 0x38, 0xe8, 0x42, + 0x8d, 0xbe, 0x80, 0xd1, 0x84, 0x86, 0x04, 0x4d, 0xc4, 0x84, 0x81, 0xc1, 0x78, 0xc5, 0x8f, 0xd2, + 0x50, 0x7a, 0xb5, 0xbd, 0x46, 0xd9, 0x7c, 0x04, 0x1f, 0xc3, 0x87, 0xf0, 0x01, 0x1c, 0x19, 0x19, + 0xe5, 0x58, 0x1c, 0x79, 0x04, 0xc3, 0x35, 0x85, 0x03, 0x4b, 0xe2, 0x04, 0xfd, 0xfd, 0x7f, 0xdf, + 0xff, 0x4b, 0xef, 0x4a, 0x0e, 0xc3, 0x9e, 0x6b, 0xf9, 0xc2, 0x0d, 0x23, 0x21, 0x85, 0xe5, 0x05, + 0x4f, 0xf0, 0xea, 0x72, 0x09, 0x2f, 0x7c, 0x50, 0xd1, 0x88, 0xee, 0x99, 0x2c, 0x74, 0x4a, 0xe5, + 0xa5, 0x81, 0xec, 0x4f, 0x2a, 0x9f, 0x7f, 0x6e, 0x90, 0xdd, 0xda, 0xcc, 0xb7, 0x53, 0x9f, 0xd6, + 0x08, 0xb9, 0x4b, 0x20, 0x1a, 0x68, 0x48, 0xcb, 0x95, 0xb9, 0xbf, 0xa0, 0x0d, 0x78, 0x4e, 0x20, + 0x96, 0xa5, 0xfd, 0xfc, 0x30, 0x0e, 0x45, 0x10, 0xc3, 0x19, 0xa6, 0x75, 0xb2, 0x63, 0x83, 0xbc, + 0xee, 0x26, 0x41, 0xaf, 0x01, 0x1d, 0x6a, 0xe8, 0x06, 0xce, 0xca, 0x0e, 0xd6, 0xa4, 0x69, 0xdb, + 0x11, 0xa2, 0x55, 0xb2, 0x6d, 0x83, 0xbc, 0x87, 0xc8, 0x83, 0x98, 0x96, 0x96, 0xec, 0x14, 0x66, + 0x4d, 0xe5, 0xdc, 0x6c, 0xde, 0xf3, 0x40, 0x0a, 0x75, 0xee, 0x80, 0x7f, 0xcb, 0xfb, 0x10, 0x57, + 0x45, 0x74, 0x03, 0x32, 0xf2, 0xda, 0xb3, 0x27, 0x7a, 0xb2, 0x98, 0x5c, 0xa3, 0x64, 0x3b, 0x0a, + 0x2b, 0xa6, 0xd1, 0xff, 0x48, 0x8a, 0x1a, 0x35, 0xb9, 0x9f, 0xac, 0x2e, 0x38, 0x5d, 0x19, 0xcb, + 0x71, 0xfe, 0xb1, 0xc1, 0x26, 0x5b, 0xb3, 0x17, 0x93, 0x5c, 0xc6, 0xe6, 0x05, 0xe9, 0xe3, 0xd7, + 0x34, 0xe7, 0x82, 0xcc, 0x70, 0x5e, 0x74, 0xa9, 0x8f, 0xb4, 0x29, 0xfc, 0xa4, 0x0f, 0xd4, 0x58, + 0x98, 0x92, 0xac, 0xa5, 0xf8, 0x37, 0xc8, 0x1a, 0xae, 0x5a, 0xc3, 0x31, 0x43, 0xa3, 0x31, 0x43, + 0xd3, 0x31, 0xc3, 0x6f, 0x8a, 0xe1, 0x0f, 0xc5, 0xf0, 0x97, 0x62, 0x78, 0xa8, 0x18, 0xfe, 0x56, + 0x0c, 0xff, 0x28, 0x86, 0xa6, 0x8a, 0xe1, 0xf7, 0x09, 0x43, 0xc3, 0x09, 0x43, 0xa3, 0x09, 0x43, + 0xad, 0x63, 0xd7, 0x93, 0xdd, 0xc4, 0xa9, 0xb4, 0x45, 0xdf, 0x72, 0x23, 0xde, 0xe1, 0x01, 0xb7, + 0x7c, 0xd1, 0xf3, 0x2c, 0xf3, 0x4b, 0x75, 0x36, 0xf5, 0xcf, 0xc5, 0x6f, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x7a, 0x1a, 0x28, 0xb4, 0xf1, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/pkg/logproto/indexgateway.proto b/pkg/logproto/indexgateway.proto index 9271ee9b2b5f4..af34e03a279cb 100644 --- a/pkg/logproto/indexgateway.proto +++ b/pkg/logproto/indexgateway.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package indexgatewaypb; -import "gogoproto/gogo.proto"; import "pkg/logproto/logproto.proto"; option go_package = "github.com/grafana/loki/pkg/logproto"; diff --git a/pkg/logproto/sketch.pb.go b/pkg/logproto/sketch.pb.go index 4a56552d984e8..c555d64d55970 100644 --- a/pkg/logproto/sketch.pb.go +++ b/pkg/logproto/sketch.pb.go @@ -7,7 +7,6 @@ import ( bytes "bytes" encoding_binary "encoding/binary" fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" io "io" math "math" @@ -657,47 +656,46 @@ func init() { func init() { proto.RegisterFile("pkg/logproto/sketch.proto", fileDescriptor_7f9fd40e59b87ff3) } var fileDescriptor_7f9fd40e59b87ff3 = []byte{ - // 632 bytes of a gzipped FileDescriptorProto + // 623 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x41, 0x4f, 0xd4, 0x4e, - 0x14, 0xef, 0xfc, 0x77, 0xff, 0xcb, 0xf2, 0x16, 0x88, 0x8e, 0xc4, 0xac, 0xc5, 0x4c, 0xd6, 0xc6, + 0x14, 0xef, 0xfc, 0x77, 0xff, 0xcb, 0xf2, 0x16, 0x88, 0x8e, 0xc4, 0xd4, 0xc5, 0x4c, 0xd6, 0xc6, 0x28, 0xd1, 0xb8, 0x9b, 0x40, 0x42, 0x38, 0x83, 0x07, 0x12, 0x45, 0x71, 0x20, 0xc6, 0x70, 0x31, - 0xa5, 0x1d, 0xba, 0x93, 0x6d, 0x3b, 0x4d, 0x67, 0x16, 0xf0, 0xe6, 0x27, 0x30, 0xc6, 0x4f, 0xe1, - 0xd5, 0x8f, 0xe0, 0xcd, 0x23, 0x47, 0x8e, 0x52, 0x2e, 0x1e, 0xf9, 0x08, 0x66, 0x66, 0xda, 0x85, - 0x2e, 0x31, 0x7a, 0xda, 0xf7, 0x7e, 0xef, 0xf7, 0x7e, 0xf3, 0x9b, 0x79, 0x7d, 0x0b, 0xf7, 0xb2, - 0x51, 0x34, 0x88, 0x45, 0x94, 0xe5, 0x42, 0x89, 0x81, 0x1c, 0x31, 0x15, 0x0c, 0xfb, 0x26, 0xc1, - 0xed, 0x0a, 0x76, 0x17, 0x23, 0x11, 0x09, 0xcb, 0xd0, 0x91, 0xad, 0xbb, 0x4b, 0xb5, 0xd6, 0x2a, - 0xb0, 0x45, 0xef, 0x15, 0x2c, 0xbe, 0x19, 0xfb, 0xa9, 0xe2, 0x31, 0xdb, 0x35, 0xa2, 0xdb, 0xbe, - 0xca, 0xf9, 0x09, 0x5e, 0x83, 0xd6, 0x91, 0x1f, 0x8f, 0x99, 0xec, 0xa2, 0x5e, 0x63, 0xb9, 0xb3, - 0x42, 0xfa, 0x93, 0xc6, 0x3a, 0xff, 0x2d, 0x0b, 0x94, 0xc8, 0x69, 0xc9, 0xf6, 0x76, 0xa6, 0xf5, - 0x6c, 0x1d, 0xaf, 0xc3, 0x8c, 0xf4, 0x93, 0x2c, 0xfe, 0xbb, 0xe0, 0xae, 0xa1, 0xd1, 0x8a, 0xee, - 0x7d, 0x42, 0xd3, 0x92, 0x96, 0x81, 0x1f, 0x01, 0x3a, 0xec, 0xa2, 0x1e, 0x5a, 0xee, 0xac, 0x74, - 0xff, 0x24, 0x46, 0xd1, 0x21, 0x7e, 0x00, 0x73, 0x8a, 0x27, 0x4c, 0x2a, 0x3f, 0xc9, 0xde, 0x27, - 0xb2, 0xfb, 0x5f, 0x0f, 0x2d, 0x37, 0x68, 0x67, 0x82, 0x6d, 0x4b, 0xfc, 0x14, 0x5a, 0x09, 0x53, - 0x39, 0x0f, 0xba, 0x0d, 0x63, 0xee, 0xce, 0x95, 0xde, 0x4b, 0xff, 0x80, 0xc5, 0x3b, 0x3e, 0xcf, - 0x69, 0x49, 0xf1, 0x22, 0x58, 0xa8, 0x1f, 0x82, 0x9f, 0xc1, 0x8c, 0x0a, 0x79, 0xc4, 0xa4, 0x2a, - 0xfd, 0xdc, 0xbe, 0xea, 0xdf, 0x7b, 0x6e, 0x0a, 0x5b, 0x0e, 0xad, 0x38, 0xf8, 0x3e, 0xb4, 0xc3, - 0xd0, 0x8e, 0xd0, 0x98, 0x99, 0xdb, 0x72, 0xe8, 0x04, 0xd9, 0x68, 0x43, 0xcb, 0x46, 0xde, 0x77, - 0x04, 0x33, 0x65, 0x3b, 0xbe, 0x05, 0x8d, 0x84, 0xa7, 0x46, 0x1e, 0x51, 0x1d, 0x1a, 0xc4, 0x3f, - 0x31, 0x02, 0x1a, 0xf1, 0x4f, 0x70, 0x0f, 0x3a, 0x81, 0x48, 0xb2, 0x9c, 0x49, 0xc9, 0x45, 0xda, - 0x6d, 0x98, 0xca, 0x75, 0x08, 0xaf, 0xc3, 0x6c, 0x96, 0x8b, 0x80, 0x49, 0xc9, 0xc2, 0x6e, 0xd3, - 0x5c, 0xd5, 0xbd, 0x61, 0xb5, 0xbf, 0xc9, 0x52, 0x95, 0x0b, 0x1e, 0xd2, 0x2b, 0xb2, 0xbb, 0x06, - 0xed, 0x0a, 0xc6, 0x18, 0x9a, 0x09, 0xf3, 0x2b, 0x33, 0x26, 0xc6, 0x77, 0xa1, 0x75, 0xcc, 0x78, - 0x34, 0x54, 0xa5, 0xa1, 0x32, 0xf3, 0xde, 0xc1, 0xc2, 0xa6, 0x18, 0xa7, 0x6a, 0x9b, 0xa7, 0xe5, - 0x63, 0x2d, 0xc2, 0xff, 0x21, 0xcb, 0xd4, 0xd0, 0xb4, 0xcf, 0x53, 0x9b, 0x68, 0xf4, 0x98, 0x87, - 0xca, 0x3e, 0xc8, 0x3c, 0xb5, 0x09, 0x76, 0xa1, 0x1d, 0xe8, 0x6e, 0x96, 0x4b, 0x33, 0x99, 0x79, - 0x3a, 0xc9, 0xbd, 0x6f, 0x08, 0x9a, 0x7b, 0x22, 0x7b, 0x81, 0x9f, 0x40, 0x23, 0x48, 0xe4, 0xcd, - 0x2f, 0xa1, 0x7e, 0x2e, 0xd5, 0x24, 0xfc, 0x18, 0x9a, 0x31, 0x97, 0xda, 0xe4, 0xd4, 0x98, 0xb5, - 0x52, 0xdf, 0x8c, 0xd9, 0x10, 0xf4, 0x5b, 0x0e, 0x3f, 0x64, 0x2c, 0x8f, 0x45, 0x14, 0x8b, 0xc8, - 0xbc, 0xe5, 0x1c, 0xbd, 0x0e, 0xb9, 0x2b, 0xd0, 0xd4, 0x7c, 0xed, 0x9c, 0x1d, 0xb1, 0xd4, 0x8e, - 0x7e, 0x96, 0xda, 0x44, 0xa3, 0xc6, 0x69, 0x75, 0x1f, 0x93, 0x78, 0x5f, 0x10, 0x80, 0x3e, 0xa9, - 0x5c, 0xb2, 0xd5, 0xa9, 0x25, 0x5b, 0xaa, 0xfb, 0xb1, 0xac, 0x7e, 0x7d, 0xc3, 0xdc, 0xd7, 0xd0, - 0x2a, 0x77, 0xca, 0x83, 0xa6, 0x12, 0xd9, 0xa8, 0xbc, 0xf9, 0x42, 0xbd, 0x99, 0x9a, 0xda, 0x3f, - 0x7c, 0xfc, 0x1b, 0xfb, 0xa7, 0xe7, 0xc4, 0x39, 0x3b, 0x27, 0xce, 0xe5, 0x39, 0x41, 0x1f, 0x0b, - 0x82, 0xbe, 0x16, 0x04, 0xfd, 0x28, 0x08, 0x3a, 0x2d, 0x08, 0xfa, 0x59, 0x10, 0xf4, 0xab, 0x20, - 0xce, 0x65, 0x41, 0xd0, 0xe7, 0x0b, 0xe2, 0x9c, 0x5e, 0x10, 0xe7, 0xec, 0x82, 0x38, 0xfb, 0x0f, - 0x23, 0xae, 0x86, 0xe3, 0x83, 0x7e, 0x20, 0x92, 0x41, 0x94, 0xfb, 0x87, 0x7e, 0xea, 0x0f, 0x62, - 0x31, 0xe2, 0x83, 0xeb, 0xff, 0x36, 0x07, 0x2d, 0xf3, 0xb3, 0xfa, 0x3b, 0x00, 0x00, 0xff, 0xff, - 0xa9, 0x7c, 0xb5, 0x30, 0xbf, 0x04, 0x00, 0x00, + 0x43, 0x3b, 0x74, 0x27, 0xdb, 0x76, 0x9a, 0xce, 0x2c, 0xe0, 0xcd, 0x4f, 0x60, 0x8c, 0x9f, 0xc2, + 0xab, 0x1f, 0xc1, 0x9b, 0x47, 0x8e, 0x1c, 0xa5, 0x5c, 0x3c, 0xf2, 0x11, 0xcc, 0x4c, 0xdb, 0x85, + 0x2e, 0x31, 0x7a, 0xda, 0x79, 0xbf, 0xf7, 0x7b, 0xbf, 0xf9, 0xcd, 0x7b, 0x7d, 0x0b, 0xf7, 0xd2, + 0x51, 0x38, 0x88, 0x64, 0x98, 0x66, 0x52, 0xcb, 0x81, 0x1a, 0x71, 0xed, 0x0f, 0xfb, 0x36, 0xc0, + 0xed, 0x0a, 0xee, 0x2e, 0xd5, 0x48, 0xd5, 0xa1, 0xa0, 0x79, 0xaf, 0x60, 0xf1, 0xcd, 0x98, 0x25, + 0x5a, 0x44, 0x7c, 0xd7, 0x96, 0x6f, 0x33, 0x9d, 0x89, 0x13, 0xbc, 0x06, 0xad, 0x23, 0x16, 0x8d, + 0xb9, 0x72, 0x51, 0xaf, 0xb1, 0xdc, 0x59, 0x21, 0xfd, 0x49, 0x61, 0x9d, 0xff, 0x96, 0xfb, 0x5a, + 0x66, 0xb4, 0x64, 0x7b, 0x3b, 0xd3, 0x7a, 0x45, 0x1e, 0xaf, 0xc3, 0x8c, 0x62, 0x71, 0x1a, 0xfd, + 0x5d, 0x70, 0xd7, 0xd2, 0x68, 0x45, 0xf7, 0x3e, 0xa1, 0x69, 0xc9, 0x82, 0x81, 0x1f, 0x01, 0x3a, + 0x74, 0x51, 0x0f, 0x2d, 0x77, 0x56, 0xdc, 0x3f, 0x89, 0x51, 0x74, 0x88, 0x1f, 0xc0, 0x9c, 0x16, + 0x31, 0x57, 0x9a, 0xc5, 0xe9, 0xfb, 0x58, 0xb9, 0xff, 0xf5, 0xd0, 0x72, 0x83, 0x76, 0x26, 0xd8, + 0xb6, 0xc2, 0x4f, 0xa1, 0x15, 0x73, 0x9d, 0x09, 0xdf, 0x6d, 0x58, 0x73, 0x77, 0xae, 0xf4, 0x5e, + 0xb2, 0x03, 0x1e, 0xed, 0x30, 0x91, 0xd1, 0x92, 0xe2, 0x85, 0xb0, 0x50, 0xbf, 0x04, 0x3f, 0x83, + 0x19, 0x1d, 0x88, 0x90, 0x2b, 0x5d, 0xfa, 0xb9, 0x7d, 0x55, 0xbf, 0xf7, 0xdc, 0x26, 0xb6, 0x1c, + 0x5a, 0x71, 0xf0, 0x7d, 0x68, 0x07, 0x41, 0x31, 0x2c, 0x6b, 0x66, 0x6e, 0xcb, 0xa1, 0x13, 0x64, + 0xa3, 0x0d, 0xad, 0xe2, 0xe4, 0x7d, 0x47, 0x30, 0x53, 0x96, 0xe3, 0x5b, 0xd0, 0x88, 0x45, 0x62, + 0xe5, 0x11, 0x35, 0x47, 0x8b, 0xb0, 0x13, 0x2b, 0x60, 0x10, 0x76, 0x82, 0x7b, 0xd0, 0xf1, 0x65, + 0x9c, 0x66, 0x5c, 0x29, 0x21, 0x13, 0xb7, 0x61, 0x33, 0xd7, 0x21, 0xbc, 0x0e, 0xb3, 0x69, 0x26, + 0x7d, 0xae, 0x14, 0x0f, 0xdc, 0xa6, 0x7d, 0x6a, 0xf7, 0x86, 0xd5, 0xfe, 0x26, 0x4f, 0x74, 0x26, + 0x45, 0x40, 0xaf, 0xc8, 0xdd, 0x35, 0x68, 0x57, 0x30, 0xc6, 0xd0, 0x8c, 0x39, 0xab, 0xcc, 0xd8, + 0x33, 0xbe, 0x0b, 0xad, 0x63, 0x2e, 0xc2, 0xa1, 0x2e, 0x0d, 0x95, 0x91, 0xf7, 0x0e, 0x16, 0x36, + 0xe5, 0x38, 0xd1, 0xdb, 0x22, 0x29, 0x9b, 0xb5, 0x08, 0xff, 0x07, 0x3c, 0xd5, 0x43, 0x5b, 0x3e, + 0x4f, 0x8b, 0xc0, 0xa0, 0xc7, 0x22, 0xd0, 0x45, 0x43, 0xe6, 0x69, 0x11, 0xe0, 0x2e, 0xb4, 0x7d, + 0x53, 0xcd, 0x33, 0x65, 0x27, 0x33, 0x4f, 0x27, 0xb1, 0xf7, 0x0d, 0x41, 0x73, 0x4f, 0xa6, 0x2f, + 0xf0, 0x13, 0x68, 0xf8, 0xb1, 0xba, 0xf9, 0x25, 0xd4, 0xef, 0xa5, 0x86, 0x84, 0x1f, 0x43, 0x33, + 0x12, 0xca, 0x98, 0x9c, 0x1a, 0xb3, 0x51, 0xea, 0xdb, 0x31, 0x5b, 0x82, 0xe9, 0xe5, 0xf0, 0x43, + 0xca, 0xb3, 0x48, 0x86, 0x91, 0x0c, 0x6d, 0x2f, 0xe7, 0xe8, 0x75, 0xa8, 0xbb, 0x02, 0x4d, 0xc3, + 0x37, 0xce, 0xf9, 0x11, 0x4f, 0x8a, 0xd1, 0xcf, 0xd2, 0x22, 0x30, 0xa8, 0x75, 0x5a, 0xbd, 0xc7, + 0x06, 0xde, 0x17, 0x04, 0x60, 0x6e, 0x2a, 0x97, 0x6c, 0x75, 0x6a, 0xc9, 0x96, 0xea, 0x7e, 0x0a, + 0x56, 0xbf, 0xbe, 0x61, 0xdd, 0xd7, 0xd0, 0x2a, 0x77, 0xca, 0x83, 0xa6, 0x96, 0xe9, 0xa8, 0x7c, + 0xf9, 0x42, 0xbd, 0x98, 0xda, 0xdc, 0x3f, 0x7c, 0xfc, 0x1b, 0xfb, 0xa7, 0xe7, 0xc4, 0x39, 0x3b, + 0x27, 0xce, 0xe5, 0x39, 0x41, 0x1f, 0x73, 0x82, 0xbe, 0xe6, 0x04, 0xfd, 0xc8, 0x09, 0x3a, 0xcd, + 0x09, 0xfa, 0x99, 0x13, 0xf4, 0x2b, 0x27, 0xce, 0x65, 0x4e, 0xd0, 0xe7, 0x0b, 0xe2, 0x9c, 0x5e, + 0x10, 0xe7, 0xec, 0x82, 0x38, 0xfb, 0x0f, 0x43, 0xa1, 0x87, 0xe3, 0x83, 0xbe, 0x2f, 0xe3, 0x41, + 0x98, 0xb1, 0x43, 0x96, 0xb0, 0x41, 0x24, 0x47, 0x62, 0x70, 0xfd, 0xdf, 0xe6, 0xa0, 0x65, 0x7f, + 0x56, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x24, 0x9c, 0x74, 0xb7, 0xa9, 0x04, 0x00, 0x00, } func (this *QuantileSketchMatrix) Equal(that interface{}) bool { diff --git a/pkg/logproto/sketch.proto b/pkg/logproto/sketch.proto index e84deaf20d4c8..d8ffeb0110340 100644 --- a/pkg/logproto/sketch.proto +++ b/pkg/logproto/sketch.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package logproto; -import "gogoproto/gogo.proto"; import "pkg/logproto/logproto.proto"; option go_package = "github.com/grafana/loki/pkg/logproto"; diff --git a/pkg/logql/blocker.go b/pkg/logql/blocker.go index cbfdc6bf49e3b..9a07113c40dd3 100644 --- a/pkg/logql/blocker.go +++ b/pkg/logql/blocker.go @@ -33,8 +33,8 @@ func (qb *queryBlocker) isBlocked(ctx context.Context, tenant string) bool { return false } - query := qb.q.params.Query() - typ, err := QueryType(query) + query := qb.q.params.QueryString() + typ, err := QueryType(qb.q.params.GetExpression()) if err != nil { typ = "unknown" } diff --git a/pkg/logql/blocker_test.go b/pkg/logql/blocker_test.go index e0dc00bf622e7..9fa586a02db80 100644 --- a/pkg/logql/blocker_test.go +++ b/pkg/logql/blocker_test.go @@ -145,15 +145,10 @@ func TestEngine_ExecWithBlockedQueries(t *testing.T) { t.Run(test.name, func(t *testing.T) { limits.blockedQueries = test.blocked - q := eng.Query(LiteralParams{ - qs: test.q, - start: time.Unix(0, 0), - end: time.Unix(100000, 0), - step: 60 * time.Second, - direction: logproto.FORWARD, - limit: 1000, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + params, err := NewLiteralParams(test.q, time.Unix(0, 0), time.Unix(100000, 0), 60*time.Second, 0, logproto.FORWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) if test.expectedErr == nil { require.NoError(t, err) diff --git a/pkg/logql/downstream.go b/pkg/logql/downstream.go index 3944b4fc492a7..2cd706c812f6a 100644 --- a/pkg/logql/downstream.go +++ b/pkg/logql/downstream.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/grafana/loki/pkg/iter" + "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/metadata" @@ -62,15 +63,12 @@ func NewDownstreamEngine(opts EngineOpts, downstreamable Downstreamable, limits func (ng *DownstreamEngine) Opts() EngineOpts { return ng.opts } // Query constructs a Query -func (ng *DownstreamEngine) Query(ctx context.Context, p Params, mapped syntax.Expr) Query { +func (ng *DownstreamEngine) Query(ctx context.Context, p Params) Query { return &query{ logger: ng.logger, params: p, evaluator: NewDownstreamEvaluator(ng.downstreamable.Downstreamer(ctx)), - parse: func(_ context.Context, _ string) (syntax.Expr, error) { - return mapped, nil - }, - limits: ng.limits, + limits: ng.limits, } } @@ -189,9 +187,7 @@ type Downstreamable interface { } type DownstreamQuery struct { - Expr syntax.Expr Params Params - Shards Shards } // Downstreamer is an interface for deferring responsibility for query execution. @@ -268,9 +264,10 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( shards = append(shards, *e.shard) } results, err := ev.Downstream(ctx, []DownstreamQuery{{ - Expr: e.SampleExpr, - Params: params, - Shards: shards, + Params: ParamsWithShardsOverride{ + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: e.SampleExpr}, + ShardsOverride: Shards(shards).Encode(), + }, }}) if err != nil { return nil, err @@ -282,11 +279,10 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( var queries []DownstreamQuery for cur != nil { qry := DownstreamQuery{ - Expr: cur.DownstreamSampleExpr.SampleExpr, - Params: params, + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: cur.DownstreamSampleExpr.SampleExpr}, } if shard := cur.DownstreamSampleExpr.shard; shard != nil { - qry.Shards = Shards{*shard} + qry.Params = ParamsWithShardsOverride{Params: qry.Params, ShardsOverride: Shards{*shard}.Encode()} } queries = append(queries, qry) cur = cur.next @@ -304,7 +300,7 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( level.Warn(util_log.Logger).Log( "msg", "could not extract StepEvaluator", "err", err, - "expr", queries[i].Expr.String(), + "expr", queries[i].Params.GetExpression().String(), ) return nil, err } @@ -332,25 +328,25 @@ func (ev *DownstreamEvaluator) NewIterator( shards = append(shards, *e.shard) } results, err := ev.Downstream(ctx, []DownstreamQuery{{ - Expr: e.LogSelectorExpr, - Params: params, - Shards: shards, + Params: ParamsWithShardsOverride{ + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: e.LogSelectorExpr}, + ShardsOverride: shards.Encode(), + }, }}) if err != nil { return nil, err } - return ResultIterator(results[0], params) + return ResultIterator(results[0], params.Direction()) case *ConcatLogSelectorExpr: cur := e var queries []DownstreamQuery for cur != nil { qry := DownstreamQuery{ - Expr: cur.DownstreamLogSelectorExpr.LogSelectorExpr, - Params: params, + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: cur.DownstreamLogSelectorExpr.LogSelectorExpr}, } if shard := cur.DownstreamLogSelectorExpr.shard; shard != nil { - qry.Shards = Shards{*shard} + qry.Params = ParamsWithShardsOverride{Params: qry.Params, ShardsOverride: Shards{*shard}.Encode()} } queries = append(queries, qry) cur = cur.next @@ -363,12 +359,12 @@ func (ev *DownstreamEvaluator) NewIterator( xs := make([]iter.EntryIterator, 0, len(results)) for i, res := range results { - iter, err := ResultIterator(res, params) + iter, err := ResultIterator(res, params.Direction()) if err != nil { level.Warn(util_log.Logger).Log( "msg", "could not extract Iterator", "err", err, - "expr", queries[i].Expr.String(), + "expr", queries[i].Params.GetExpression().String(), ) } xs = append(xs, iter) @@ -452,10 +448,10 @@ func NewResultStepEvaluator(res logqlmodel.Result, params Params) (StepEvaluator } // ResultIterator coerces a downstream streams result into an iter.EntryIterator -func ResultIterator(res logqlmodel.Result, params Params) (iter.EntryIterator, error) { +func ResultIterator(res logqlmodel.Result, direction logproto.Direction) (iter.EntryIterator, error) { streams, ok := res.Data.(logqlmodel.Streams) if !ok { return nil, fmt.Errorf("unexpected type (%s) for ResultIterator; expected %s", res.Data.Type(), logqlmodel.ValueTypeStreams) } - return iter.NewStreamsIterator(streams, params.Direction()), nil + return iter.NewStreamsIterator(streams, direction), nil } diff --git a/pkg/logql/downstream_test.go b/pkg/logql/downstream_test.go index c5f54b9e1c056..0f4d1cd09984d 100644 --- a/pkg/logql/downstream_test.go +++ b/pkg/logql/downstream_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" ) var nilShardMetrics = NewShardMapperMetrics(nil) @@ -69,7 +70,7 @@ func TestMappingEquivalence(t *testing.T) { sharded := NewDownstreamEngine(opts, MockDownstreamer{regular}, NoLimits, log.NewNopLogger()) t.Run(tc.query, func(t *testing.T) { - params := NewLiteralParams( + params, err := NewLiteralParams( tc.query, start, end, @@ -79,14 +80,16 @@ func TestMappingEquivalence(t *testing.T) { uint32(limit), nil, ) + require.NoError(t, err) + qry := regular.Query(params) ctx := user.InjectOrgID(context.Background(), "fake") mapper := NewShardMapper(ConstantShards(shards), nilShardMetrics) - _, _, mapped, err := mapper.Parse(tc.query) + _, _, mapped, err := mapper.Parse(params.GetExpression()) require.Nil(t, err) - shardedQry := sharded.Query(ctx, params, mapped) + shardedQry := sharded.Query(ctx, ParamsWithExpressionOverride{Params: params, ExpressionOverride: mapped}) res, err := qry.Exec(ctx) require.Nil(t, err) @@ -135,7 +138,7 @@ func TestShardCounter(t *testing.T) { sharded := NewDownstreamEngine(opts, MockDownstreamer{regular}, NoLimits, log.NewNopLogger()) t.Run(tc.query, func(t *testing.T) { - params := NewLiteralParams( + params, err := NewLiteralParams( tc.query, start, end, @@ -145,13 +148,14 @@ func TestShardCounter(t *testing.T) { uint32(limit), nil, ) + require.NoError(t, err) ctx := user.InjectOrgID(context.Background(), "fake") mapper := NewShardMapper(ConstantShards(shards), nilShardMetrics) - noop, _, mapped, err := mapper.Parse(tc.query) - require.Nil(t, err) + noop, _, mapped, err := mapper.Parse(params.GetExpression()) + require.NoError(t, err) - shardedQry := sharded.Query(ctx, params, mapped) + shardedQry := sharded.Query(ctx, ParamsWithExpressionOverride{Params: params, ExpressionOverride: mapped}) shardedRes, err := shardedQry.Exec(ctx) require.Nil(t, err) @@ -393,7 +397,7 @@ func TestRangeMappingEquivalence(t *testing.T) { t.Run(tc.query, func(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "fake") - params := NewLiteralParams( + params, err := NewLiteralParams( tc.query, start, end, @@ -403,6 +407,7 @@ func TestRangeMappingEquivalence(t *testing.T) { uint32(limit), nil, ) + require.NoError(t, err) // Regular engine qry := regularEngine.Query(params) @@ -412,12 +417,12 @@ func TestRangeMappingEquivalence(t *testing.T) { // Downstream engine - split by range rangeMapper, err := NewRangeMapper(tc.splitByInterval, nilRangeMetrics, NewMapperStats()) require.Nil(t, err) - noop, rangeExpr, err := rangeMapper.Parse(tc.query) + noop, rangeExpr, err := rangeMapper.Parse(syntax.MustParseExpr(tc.query)) require.Nil(t, err) require.False(t, noop, "downstream engine cannot execute noop") - rangeQry := downstreamEngine.Query(ctx, params, rangeExpr) + rangeQry := downstreamEngine.Query(ctx, ParamsWithExpressionOverride{Params: params, ExpressionOverride: rangeExpr}) rangeRes, err := rangeQry.Exec(ctx) require.Nil(t, err) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index af680a33b9a97..e04cf1dcffa71 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -160,12 +160,9 @@ func NewEngine(opts EngineOpts, q Querier, l Limits, logger log.Logger) *Engine // Query creates a new LogQL query. Instant/Range type is derived from the parameters. func (ng *Engine) Query(params Params) Query { return &query{ - logger: ng.logger, - params: params, - evaluator: ng.evaluatorFactory, - parse: func(_ context.Context, query string) (syntax.Expr, error) { - return syntax.ParseExpr(query) - }, + logger: ng.logger, + params: params, + evaluator: ng.evaluatorFactory, record: true, logExecQuery: ng.opts.LogExecutingQuery, limits: ng.limits, @@ -181,7 +178,6 @@ type Query interface { type query struct { logger log.Logger params Params - parse func(context.Context, string) (syntax.Expr, error) limits Limits evaluator EvaluatorFactory record bool @@ -211,7 +207,7 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) { sp.LogKV( "type", GetRangeType(q.params), - "query", q.params.Query(), + "query", q.params.QueryString(), "start", q.params.Start(), "end", q.params.End(), "step", q.params.Step(), @@ -219,11 +215,11 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) { ) if q.logExecQuery { - queryHash := util.HashedQuery(q.params.Query()) + queryHash := util.HashedQuery(q.params.QueryString()) if GetRangeType(q.params) == InstantType { - level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "instant", "query", q.params.Query(), "query_hash", queryHash) + level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "instant", "query", q.params.QueryString(), "query_hash", queryHash) } else { - level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "range", "query", q.params.Query(), "length", q.params.End().Sub(q.params.Start()), "step", q.params.Step(), "query_hash", queryHash) + level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "range", "query", q.params.QueryString(), "length", q.params.End().Sub(q.params.Start()), "step", q.params.Step(), "query_hash", queryHash) } } @@ -263,16 +259,11 @@ func (q *query) Eval(ctx context.Context) (promql_parser.Value, error) { ctx, cancel := context.WithTimeout(ctx, queryTimeout) defer cancel() - expr, err := q.parse(ctx, q.params.Query()) - if err != nil { - return nil, err - } - if q.checkBlocked(ctx, tenants) { return nil, logqlmodel.ErrBlocked } - switch e := expr.(type) { + switch e := q.params.GetExpression().(type) { case syntax.SampleExpr: value, err := q.evalSample(ctx, e) return value, err @@ -364,7 +355,7 @@ func (q *query) evalSample(ctx context.Context, expr syntax.SampleExpr) (promql_ if GetRangeType(q.params) == InstantType { sortByValue, err := Sortable(q.params) if err != nil { - return nil, fmt.Errorf("fail to check Sortable, logql: %s ,err: %s", q.params.Query(), err) + return nil, fmt.Errorf("fail to check Sortable, logql: %s ,err: %s", q.params.QueryString(), err) } if !sortByValue { sort.Slice(vec, func(i, j int) bool { return labels.Compare(vec[i].Metric, vec[j].Metric) < 0 }) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 548400644a31e..e0b6ab3dff2ae 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -129,13 +129,9 @@ func TestEngine_LogsRateUnwrap(t *testing.T) { t.Parallel() eng := NewEngine(EngineOpts{}, newQuerierRecorder(t, test.data, test.params), NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: test.qs, - start: test.ts, - end: test.ts, - direction: test.direction, - limit: test.limit, - }) + params, err := NewLiteralParams(test.qs, test.ts, test.ts, 0, 0, test.direction, test.limit, nil) + require.NoError(t, err) + q := eng.Query(params) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if expectedError, ok := test.expected.(error); ok { assert.Equal(t, expectedError.Error(), err.Error()) @@ -960,13 +956,10 @@ func TestEngine_LogsInstantQuery(t *testing.T) { t.Parallel() eng := NewEngine(EngineOpts{}, newQuerierRecorder(t, test.data, test.params), NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: test.qs, - start: test.ts, - end: test.ts, - direction: test.direction, - limit: test.limit, - }) + + params, err := NewLiteralParams(test.qs, test.ts, test.ts, 0, 0, test.direction, test.limit, nil) + require.NoError(t, err) + q := eng.Query(params) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if expectedError, ok := test.expected.(error); ok { assert.Equal(t, expectedError.Error(), err.Error()) @@ -2266,15 +2259,9 @@ func TestEngine_RangeQuery(t *testing.T) { eng := NewEngine(EngineOpts{}, newQuerierRecorder(t, test.data, test.params), NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: test.qs, - start: test.start, - end: test.end, - step: test.step, - interval: test.interval, - direction: test.direction, - limit: test.limit, - }) + params, err := NewLiteralParams(test.qs, test.start, test.end, test.step, test.interval, test.direction, test.limit, nil) + require.NoError(t, err) + q := eng.Query(params) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if err != nil { t.Fatal(err) @@ -2302,13 +2289,11 @@ func TestEngine_Stats(t *testing.T) { eng := NewEngine(EngineOpts{}, &statsQuerier{}, NoLimits, log.NewNopLogger()) queueTime := 2 * time.Nanosecond - q := eng.Query(LiteralParams{ - qs: `{foo="bar"}`, - start: time.Now(), - end: time.Now(), - direction: logproto.BACKWARD, - limit: 1000, - }) + + params, err := NewLiteralParams(`{foo="bar"}`, time.Now(), time.Now(), 0, 0, logproto.FORWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + ctx := context.WithValue(context.Background(), httpreq.QueryQueueTimeHTTPHeader, queueTime) r, err := q.Exec(user.InjectOrgID(ctx, "fake")) require.NoError(t, err) @@ -2338,13 +2323,9 @@ func (metaQuerier) SelectSamples(ctx context.Context, _ SelectSampleParams) (ite func TestEngine_Metadata(t *testing.T) { eng := NewEngine(EngineOpts{}, &metaQuerier{}, NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: `{foo="bar"}`, - start: time.Now(), - end: time.Now(), - direction: logproto.BACKWARD, - limit: 1000, - }) + params, err := NewLiteralParams(`{foo="bar"}`, time.Now(), time.Now(), 0, 0, logproto.BACKWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) r, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) require.NoError(t, err) @@ -2353,51 +2334,17 @@ func TestEngine_Metadata(t *testing.T) { }, r.Headers) } -func TestEngine_LogsInstantQuery_IllegalLogql(t *testing.T) { - eng := NewEngine(EngineOpts{}, &statsQuerier{}, NoLimits, log.NewNopLogger()) - - queueTime := 2 * time.Nanosecond - illegalVector := `vector(abc)` - q := eng.Query(LiteralParams{ - qs: illegalVector, - start: time.Now(), - end: time.Now(), - step: time.Second * 30, - interval: time.Second * 30, - direction: logproto.BACKWARD, - limit: 1000, - }) - expectErr := logqlmodel.NewParseError("syntax error: unexpected IDENTIFIER, expecting NUMBER", 1, 8) - ctx := context.WithValue(context.Background(), httpreq.QueryQueueTimeHTTPHeader, queueTime) - _, err := q.Exec(user.InjectOrgID(ctx, "fake")) - - require.EqualError(t, err, expectErr.Error()) - - qry, ok := q.(*query) - require.Equal(t, ok, true) - vectorExpr := syntax.NewVectorExpr(illegalVector) - - _, err = qry.evalSample(ctx, vectorExpr) - expectEvalSampleErr := logqlmodel.NewParseError("unable to parse vectorExpr as a float: strconv.ParseFloat: parsing \"vector(abc)\": invalid syntax", 0, 0) - require.EqualError(t, err, expectEvalSampleErr.Error()) -} - func TestEngine_LogsInstantQuery_Vector(t *testing.T) { eng := NewEngine(EngineOpts{}, &statsQuerier{}, NoLimits, log.NewNopLogger()) now := time.Now() queueTime := 2 * time.Nanosecond logqlVector := `vector(5)` - q := eng.Query(LiteralParams{ - qs: logqlVector, - start: now, - end: now, - step: 0, - interval: time.Second * 30, - direction: logproto.BACKWARD, - limit: 1000, - }) + + params, err := NewLiteralParams(logqlVector, now, now, 0, time.Second*30, logproto.BACKWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) ctx := context.WithValue(context.Background(), httpreq.QueryQueueTimeHTTPHeader, queueTime) - _, err := q.Exec(user.InjectOrgID(ctx, "fake")) + _, err = q.Exec(user.InjectOrgID(ctx, "fake")) require.NoError(t, err) @@ -2472,14 +2419,11 @@ func TestStepEvaluator_Error(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { eng := NewEngine(EngineOpts{}, tc.querier, NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: tc.qs, - start: time.Unix(0, 0), - end: time.Unix(180, 0), - step: 1 * time.Second, - limit: 1, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + + params, err := NewLiteralParams(tc.qs, time.Unix(0, 0), time.Unix(180, 0), 1*time.Second, 0, logproto.BACKWARD, 1, nil) + require.NoError(t, err) + q := eng.Query(params) + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) require.Equal(t, tc.err, err) }) } @@ -2502,15 +2446,10 @@ func TestEngine_MaxSeries(t *testing.T) { {`avg(count_over_time({app=~"foo|bar"} |~".+bar" [1m]))`, logproto.FORWARD, false}, } { t.Run(test.qs, func(t *testing.T) { - q := eng.Query(LiteralParams{ - qs: test.qs, - start: time.Unix(0, 0), - end: time.Unix(100000, 0), - step: 60 * time.Second, - direction: test.direction, - limit: 1000, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + params, err := NewLiteralParams(test.qs, time.Unix(0, 0), time.Unix(100000, 0), 60*time.Second, 0, test.direction, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) if test.expectLimitErr { require.NotNil(t, err) require.True(t, errors.Is(err, logqlmodel.ErrLimit)) @@ -2534,15 +2473,11 @@ func TestEngine_MaxRangeInterval(t *testing.T) { {`topk(1,rate({app=~"foo|bar"}[12h]) / (rate({app="baz"}[23h]) + rate({app="fiz"}[25h])))`, logproto.FORWARD, true}, } { t.Run(test.qs, func(t *testing.T) { - q := eng.Query(LiteralParams{ - qs: test.qs, - start: time.Unix(0, 0), - end: time.Unix(100000, 0), - step: 60 * time.Second, - direction: test.direction, - limit: 1000, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + params, err := NewLiteralParams(test.qs, time.Unix(0, 0), time.Unix(100000, 0), 60*time.Second, 0, test.direction, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) if test.expectLimitErr { require.Error(t, err) require.ErrorIs(t, err, logqlmodel.ErrIntervalLimit) @@ -2605,14 +2540,10 @@ func benchmarkRangeQuery(testsize int64, b *testing.B) { {`bottomk(2,rate(({app=~"foo|bar"} |~".+bar")[1m]))`, logproto.FORWARD}, {`bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app)`, logproto.FORWARD}, } { - q := eng.Query(LiteralParams{ - qs: test.qs, - start: start, - end: end, - step: 60 * time.Second, - direction: test.direction, - limit: 1000, - }) + params, err := NewLiteralParams(test.qs, start, end, 60*time.Second, 0, logproto.BACKWARD, 1000, nil) + require.NoError(b, err) + q := eng.Query(params) + res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if err != nil { b.Fatal(err) @@ -2640,8 +2571,13 @@ func TestHashingStability(t *testing.T) { buf := bytes.NewBufferString("") logger := log.NewLogfmtLogger(buf) eng := NewEngine(EngineOpts{LogExecutingQuery: true}, getLocalQuerier(4), NoLimits, logger) + + parsed, err := syntax.ParseExpr(params.QueryString()) + require.NoError(t, err) + params.queryExpr = parsed + query := eng.Query(params) - _, err := query.Exec(ctx) + _, err = query.Exec(ctx) require.NoError(t, err) return buf.String() } @@ -2668,7 +2604,7 @@ func TestHashingStability(t *testing.T) { {`sum by(query_hash) (count_over_time({app="myapp",env="myenv"} |= "error" |= "metrics.go" | logfmt [10s]))`}, {`sum (count_over_time({app="myapp",env="myenv"} |= "error" |= "metrics.go" | logfmt [10s])) by(query_hash)`}, } { - params.qs = test.qs + params.queryString = test.qs expectedQueryHash := util.HashedQuery(test.qs) // check that both places will end up having the same query hash, even though they're emitting different log lines. diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 0c0dba2cad3d5..fdb9190956a5b 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -31,7 +31,7 @@ var ( // Params details the parameters associated with a loki request type Params interface { - Query() string + QueryString() string Start() time.Time End() time.Time Step() time.Duration @@ -39,6 +39,7 @@ type Params interface { Limit() uint32 Direction() logproto.Direction Shards() []string + GetExpression() syntax.Expr } func NewLiteralParams( @@ -48,33 +49,41 @@ func NewLiteralParams( direction logproto.Direction, limit uint32, shards []string, -) LiteralParams { - return LiteralParams{ - qs: qs, - start: start, - end: end, - step: step, - interval: interval, - direction: direction, - limit: limit, - shards: shards, - } +) (LiteralParams, error) { + p := LiteralParams{ + queryString: qs, + start: start, + end: end, + step: step, + interval: interval, + direction: direction, + limit: limit, + shards: shards, + } + var err error + p.queryExpr, err = syntax.ParseExpr(qs) + return p, err + } // LiteralParams impls Params type LiteralParams struct { - qs string + queryString string start, end time.Time step, interval time.Duration direction logproto.Direction limit uint32 shards []string + queryExpr syntax.Expr } func (p LiteralParams) Copy() LiteralParams { return p } // String impls Params -func (p LiteralParams) Query() string { return p.qs } +func (p LiteralParams) QueryString() string { return p.queryString } + +// GetExpression impls Params +func (p LiteralParams) GetExpression() syntax.Expr { return p.queryExpr } // Start impls Params func (p LiteralParams) Start() time.Time { return p.start } @@ -105,12 +114,38 @@ func GetRangeType(q Params) QueryRangeType { return RangeType } +// ParamsWithExpressionOverride overrides the query expression so that the query +// string and the expression can differ. This is useful for for query planning +// when plan my not match externally available logql syntax +type ParamsWithExpressionOverride struct { + Params + ExpressionOverride syntax.Expr +} + +// GetExpression returns the parsed expression of the query. +func (p ParamsWithExpressionOverride) GetExpression() syntax.Expr { + return p.ExpressionOverride +} + +// ParamsWithExpressionOverride overrides the shards. Since the backing +// implementation of the Params interface is unknown they are embedded and the +// original shards are shadowed. +type ParamsWithShardsOverride struct { + Params + ShardsOverride []string +} + +// Shards returns this overwriting shards. +func (p ParamsWithShardsOverride) Shards() []string { + return p.ShardsOverride +} + // Sortable logql contain sort or sort_desc. func Sortable(q Params) (bool, error) { var sortable bool - expr, err := syntax.ParseSampleExpr(q.Query()) - if err != nil { - return false, err + expr, ok := q.GetExpression().(syntax.SampleExpr) + if !ok { + return false, errors.New("only sample expression supported") } expr.Walk(func(e syntax.Expr) { rangeExpr, ok := e.(*syntax.VectorAggregationExpr) diff --git a/pkg/logql/evaluator_test.go b/pkg/logql/evaluator_test.go index 1bec3d9c67d68..e31d587252066 100644 --- a/pkg/logql/evaluator_test.go +++ b/pkg/logql/evaluator_test.go @@ -44,14 +44,14 @@ func TestDefaultEvaluator_DivideByZero(t *testing.T) { } func TestDefaultEvaluator_Sortable(t *testing.T) { logqlSort := `sort(rate(({app=~"foo|bar"} |~".+bar")[1m])) ` - sortable, err := Sortable(LiteralParams{qs: logqlSort}) + sortable, err := Sortable(LiteralParams{queryString: logqlSort, queryExpr: syntax.MustParseExpr(logqlSort)}) if err != nil { t.Fatal(err) } require.Equal(t, true, sortable) logqlSum := `sum(rate(({app=~"foo|bar"} |~".+bar")[1m])) ` - sortableSum, err := Sortable(LiteralParams{qs: logqlSum}) + sortableSum, err := Sortable(LiteralParams{queryString: logqlSum, queryExpr: syntax.MustParseExpr(logqlSum)}) if err != nil { t.Fatal(err) } diff --git a/pkg/logql/explain_test.go b/pkg/logql/explain_test.go index a54ffa5916f2c..5ae2f840e1c88 100644 --- a/pkg/logql/explain_test.go +++ b/pkg/logql/explain_test.go @@ -29,14 +29,14 @@ func TestExplain(t *testing.T) { downEv := &DownstreamEvaluator{Downstreamer: MockDownstreamer{regular}, defaultEvaluator: defaultEv} mapper := NewShardMapper(ConstantShards(4), nilShardMetrics) - _, _, expr, err := mapper.Parse(query) + _, _, expr, err := mapper.Parse(syntax.MustParseExpr(query)) require.NoError(t, err) params := LiteralParams{ - qs: query, - start: time.Unix(60, 0), - end: time.Unix(60, 0), - limit: 1000, + queryString: query, + start: time.Unix(60, 0), + end: time.Unix(60, 0), + limit: 1000, } ev, err := downEv.NewStepEvaluator(ctx, downEv, expr.(syntax.SampleExpr), params) diff --git a/pkg/logql/metrics.go b/pkg/logql/metrics.go index 94a4c2f9dd408..9db8ee96e4ed4 100644 --- a/pkg/logql/metrics.go +++ b/pkg/logql/metrics.go @@ -98,7 +98,7 @@ func RecordRangeAndInstantQueryMetrics( latencyType = latencyTypeFast returnedLines = 0 ) - queryType, err := QueryType(p.Query()) + queryType, err := QueryType(p.GetExpression()) if err != nil { level.Warn(logger).Log("msg", "error parsing query type", "err", err) } @@ -119,8 +119,8 @@ func RecordRangeAndInstantQueryMetrics( logValues = append(logValues, []interface{}{ "latency", latencyType, // this can be used to filter log lines. - "query", p.Query(), - "query_hash", util.HashedQuery(p.Query()), + "query", p.QueryString(), + "query_hash", util.HashedQuery(p.QueryString()), "query_type", queryType, "range_type", rt, "length", p.End().Sub(p.Start()), @@ -373,11 +373,7 @@ func recordUsageStats(queryType string, stats logql_stats.Result) { } } -func QueryType(query string) (string, error) { - expr, err := syntax.ParseExpr(query) - if err != nil { - return "", err - } +func QueryType(expr syntax.Expr) (string, error) { switch e := expr.(type) { case syntax.SampleExpr: return QueryTypeMetric, nil diff --git a/pkg/logql/metrics_test.go b/pkg/logql/metrics_test.go index 06d4e2699494e..6d07040bb802a 100644 --- a/pkg/logql/metrics_test.go +++ b/pkg/logql/metrics_test.go @@ -16,6 +16,7 @@ import ( "github.com/uber/jaeger-client-go" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" "github.com/grafana/loki/pkg/util" @@ -25,30 +26,25 @@ import ( func TestQueryType(t *testing.T) { tests := []struct { - name string - query string - want string - wantErr bool + name string + query string + want string }{ - {"bad", "ddd", "", true}, - {"limited", `{app="foo"}`, QueryTypeLimited, false}, - {"limited multi label", `{app="foo" ,fuzz=~"foo"}`, QueryTypeLimited, false}, - {"limited with parser", `{app="foo" ,fuzz=~"foo"} | logfmt`, QueryTypeLimited, false}, - {"filter", `{app="foo"} |= "foo"`, QueryTypeFilter, false}, - {"filter string extracted label", `{app="foo"} | json | foo="a"`, QueryTypeFilter, false}, - {"filter duration", `{app="foo"} | json | duration > 5s`, QueryTypeFilter, false}, - {"metrics", `rate({app="foo"} |= "foo"[5m])`, QueryTypeMetric, false}, - {"metrics binary", `rate({app="foo"} |= "foo"[5m]) + count_over_time({app="foo"} |= "foo"[5m]) / rate({app="foo"} |= "foo"[5m]) `, QueryTypeMetric, false}, - {"filters", `{app="foo"} |= "foo" |= "f" != "b"`, QueryTypeFilter, false}, - {"filters and labels filters", `{app="foo"} |= "foo" |= "f" != "b" | json | a > 5`, QueryTypeFilter, false}, + {"limited", `{app="foo"}`, QueryTypeLimited}, + {"limited multi label", `{app="foo" ,fuzz=~"foo"}`, QueryTypeLimited}, + {"limited with parser", `{app="foo" ,fuzz=~"foo"} | logfmt`, QueryTypeLimited}, + {"filter", `{app="foo"} |= "foo"`, QueryTypeFilter}, + {"filter string extracted label", `{app="foo"} | json | foo="a"`, QueryTypeFilter}, + {"filter duration", `{app="foo"} | json | duration > 5s`, QueryTypeFilter}, + {"metrics", `rate({app="foo"} |= "foo"[5m])`, QueryTypeMetric}, + {"metrics binary", `rate({app="foo"} |= "foo"[5m]) + count_over_time({app="foo"} |= "foo"[5m]) / rate({app="foo"} |= "foo"[5m]) `, QueryTypeMetric}, + {"filters", `{app="foo"} |= "foo" |= "f" != "b"`, QueryTypeFilter}, + {"filters and labels filters", `{app="foo"} |= "foo" |= "f" != "b" | json | a > 5`, QueryTypeFilter}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := QueryType(tt.query) - if (err != nil) != tt.wantErr { - t.Errorf("QueryType() error = %v, wantErr %v", err, tt.wantErr) - return - } + got, err := QueryType(syntax.MustParseExpr(tt.query)) + require.NoError(t, err) if got != tt.want { t.Errorf("QueryType() = %v, want %v", got, tt.want) } @@ -69,12 +65,13 @@ func TestLogSlowQuery(t *testing.T) { ctx = context.WithValue(ctx, httpreq.QueryTagsHTTPHeader, "Source=logvolhist,Feature=Beta") RecordRangeAndInstantQueryMetrics(ctx, util_log.Logger, LiteralParams{ - qs: `{foo="bar"} |= "buzz"`, - direction: logproto.BACKWARD, - end: now, - start: now.Add(-1 * time.Hour), - limit: 1000, - step: time.Minute, + queryString: `{foo="bar"} |= "buzz"`, + direction: logproto.BACKWARD, + end: now, + start: now.Add(-1 * time.Hour), + limit: 1000, + step: time.Minute, + queryExpr: syntax.MustParseExpr(`{foo="bar"} |= "buzz"`), }, "200", stats.Result{ Summary: stats.Summary{ BytesProcessedPerSecond: 100000, diff --git a/pkg/logql/rangemapper.go b/pkg/logql/rangemapper.go index cc63944bc07e9..250f586603b7e 100644 --- a/pkg/logql/rangemapper.go +++ b/pkg/logql/rangemapper.go @@ -81,10 +81,10 @@ func NewRangeMapperMetrics(registerer prometheus.Registerer) *MapperMetrics { // be executed by the downstream engine. // It returns a boolean indicating whether a rewrite was possible, the // rewritten sample expression, and an error in case the rewrite failed. -func (m RangeMapper) Parse(query string) (bool, syntax.Expr, error) { - origExpr, err := syntax.ParseSampleExpr(query) - if err != nil { - return true, nil, err +func (m RangeMapper) Parse(expr syntax.Expr) (bool, syntax.Expr, error) { + origExpr, ok := expr.(syntax.SampleExpr) + if !ok { + return true, nil, errors.New("only sample expression supported") } recorder := m.metrics.downstreamRecorder() diff --git a/pkg/logql/rangemapper_test.go b/pkg/logql/rangemapper_test.go index 1c2f827867f93..48394d219be1a 100644 --- a/pkg/logql/rangemapper_test.go +++ b/pkg/logql/rangemapper_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/logql/syntax" ) func Test_SplitRangeInterval(t *testing.T) { @@ -83,7 +83,7 @@ func Test_SplitRangeInterval(t *testing.T) { rvm, err := NewRangeMapper(2*time.Second, nilShardMetrics, mapperStats) require.NoError(t, err) - noop, mappedExpr, err := rvm.Parse(tc.expr) + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) require.NoError(t, err) require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) @@ -1741,7 +1741,7 @@ func Test_SplitRangeVectorMapping(t *testing.T) { rvm, err := NewRangeMapper(time.Minute, nilShardMetrics, mapperStats) require.NoError(t, err) - noop, mappedExpr, err := rvm.Parse(tc.expr) + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) require.NoError(t, err) require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) @@ -1932,7 +1932,7 @@ func Test_SplitRangeVectorMapping_Noop(t *testing.T) { rvm, err := NewRangeMapper(time.Minute, nilShardMetrics, mapperStats) require.NoError(t, err) - noop, mappedExpr, err := rvm.Parse(tc.expr) + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) require.NoError(t, err) require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) @@ -1945,21 +1945,9 @@ func Test_SplitRangeVectorMapping_Noop(t *testing.T) { func Test_FailQuery(t *testing.T) { rvm, err := NewRangeMapper(2*time.Minute, nilShardMetrics, NewMapperStats()) require.NoError(t, err) - _, _, err = rvm.Parse(`{app="foo"} |= "err"`) + _, _, err = rvm.Parse(syntax.MustParseExpr(`{app="foo"} |= "err"`)) require.Error(t, err) - _, _, err = rvm.Parse(`topk(0, sum(count_over_time({app="foo"} | json | __error__="" [15m])))`) - require.Error(t, err) - // Check fixes for bug where missing or empty parameters for regexp and pattern parsers threw a panic - // Missing parameter to regexp parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | regexp [1d])))`) - require.ErrorIs(t, err, logqlmodel.ErrParse) - // Empty parameter to regexp parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | regexp ` + "``" + ` [1d])))`) - require.ErrorIs(t, err, logqlmodel.ErrParse) - // Empty parameter to pattern parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | pattern ` + `""` + ` [1d])))`) - require.ErrorIs(t, err, logqlmodel.ErrParse) // Empty parameter to json parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | json [1d])))`) + _, _, err = rvm.Parse(syntax.MustParseExpr(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | json [1d])))`)) require.NoError(t, err) } diff --git a/pkg/logql/shardmapper.go b/pkg/logql/shardmapper.go index 6409cdbf0860c..f1ee7e4ba6985 100644 --- a/pkg/logql/shardmapper.go +++ b/pkg/logql/shardmapper.go @@ -41,12 +41,7 @@ func NewShardMapperMetrics(registerer prometheus.Registerer) *MapperMetrics { return newMapperMetrics(registerer, "shard") } -func (m ShardMapper) Parse(query string) (noop bool, bytesPerShard uint64, expr syntax.Expr, err error) { - parsed, err := syntax.ParseExpr(query) - if err != nil { - return false, 0, nil, err - } - +func (m ShardMapper) Parse(parsed syntax.Expr) (noop bool, bytesPerShard uint64, expr syntax.Expr, err error) { recorder := m.metrics.downstreamRecorder() mapped, bytesPerShard, err := m.Map(parsed, recorder) diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index bdfd8a6c42d41..80b2e68751fee 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -1361,14 +1361,14 @@ func mustNewMatcher(t labels.MatchType, n, v string) *labels.Matcher { func TestStringTrimming(t *testing.T) { for _, tc := range []struct { - expr string + expr syntax.Expr expected string shards int }{ { // sample expr in entirety for low shard count shards: 2, - expr: `count_over_time({app="foo"}[1m])`, + expr: syntax.MustParseExpr(`count_over_time({app="foo"}[1m])`), expected: ` downstream ++ downstream @@ -1377,7 +1377,7 @@ func TestStringTrimming(t *testing.T) { { // sample expr doesnt display infinite shards shards: 5, - expr: `count_over_time({app="foo"}[1m])`, + expr: syntax.MustParseExpr(`count_over_time({app="foo"}[1m])`), expected: ` downstream ++ downstream ++ @@ -1389,7 +1389,7 @@ func TestStringTrimming(t *testing.T) { { // log selector expr in entirety for low shard count shards: 2, - expr: `{app="foo"}`, + expr: syntax.MustParseExpr(`{app="foo"}`), expected: ` downstream<{app="foo"},shard=0_of_2> ++ downstream<{app="foo"},shard=1_of_2> @@ -1398,7 +1398,7 @@ func TestStringTrimming(t *testing.T) { { // log selector expr doesnt display infinite shards shards: 5, - expr: `{app="foo"}`, + expr: syntax.MustParseExpr(`{app="foo"}`), expected: ` downstream<{app="foo"},shard=0_of_5> ++ downstream<{app="foo"},shard=1_of_5> ++ @@ -1408,7 +1408,7 @@ func TestStringTrimming(t *testing.T) { `, }, } { - t.Run(tc.expr, func(t *testing.T) { + t.Run(tc.expr.String(), func(t *testing.T) { m := NewShardMapper(ConstantShards(tc.shards), nilShardMetrics) _, _, mappedExpr, err := m.Parse(tc.expr) require.Nil(t, err) diff --git a/pkg/logql/syntax/parser.go b/pkg/logql/syntax/parser.go index e1fe5971ff3a2..81874ba6d6c41 100644 --- a/pkg/logql/syntax/parser.go +++ b/pkg/logql/syntax/parser.go @@ -99,6 +99,14 @@ func ParseExprWithoutValidation(input string) (expr Expr, err error) { return p.Parse() } +func MustParseExpr(input string) Expr { + expr, err := ParseExpr(input) + if err != nil { + panic(err) + } + return expr +} + func validateExpr(expr Expr) error { switch e := expr.(type) { case SampleExpr: diff --git a/pkg/logql/test_utils.go b/pkg/logql/test_utils.go index 982fa7f5f16d0..b979dedb42327 100644 --- a/pkg/logql/test_utils.go +++ b/pkg/logql/test_utils.go @@ -218,17 +218,7 @@ func (m MockDownstreamer) Downstreamer(_ context.Context) Downstreamer { return func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery) ([]logqlmodel.Result, error) { results := make([]logqlmodel.Result, 0, len(queries)) for _, query := range queries { - params := NewLiteralParams( - query.Expr.String(), - query.Params.Start(), - query.Params.End(), - query.Params.Step(), - query.Params.Interval(), - query.Params.Direction(), - query.Params.Limit(), - query.Shards.Encode(), - ) - res, err := m.Query(params).Exec(ctx) + res, err := m.Query(query.Params).Exec(ctx) if err != nil { return nil, err } diff --git a/pkg/querier/plan/plan.go b/pkg/querier/plan/plan.go new file mode 100644 index 0000000000000..6822932d7b241 --- /dev/null +++ b/pkg/querier/plan/plan.go @@ -0,0 +1,101 @@ +package plan + +import ( + "bytes" + + "github.com/grafana/loki/pkg/logql/syntax" +) + +type QueryPlan struct { + AST syntax.Expr +} + +func (t QueryPlan) Marshal() ([]byte, error) { + return t.MarshalJSON() +} + +func (t *QueryPlan) MarshalTo(data []byte) (int, error) { + appender := &appendWriter{ + slice: data[:0], + } + err := syntax.EncodeJSON(t.AST, appender) + if err != nil { + return 0, err + } + + return len(appender.slice), nil +} + +func (t *QueryPlan) Unmarshal(data []byte) error { + return t.UnmarshalJSON(data) +} + +func (t *QueryPlan) Size() int { + counter := &countWriter{} + err := syntax.EncodeJSON(t.AST, counter) + if err != nil { + return 0 + } + + return counter.bytes +} + +func (t QueryPlan) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + err := syntax.EncodeJSON(t.AST, &buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (t *QueryPlan) UnmarshalJSON(data []byte) error { + // An empty query plan is ingored to be backwards compatible. + if len(data) == 0 { + return nil + } + + expr, err := syntax.DecodeJSON(string(data)) + if err != nil { + return err + } + + t.AST = expr + return nil +} + +func (t QueryPlan) Equal(other QueryPlan) bool { + left, err := t.Marshal() + if err != nil { + return false + } + + right, err := other.Marshal() + if err != nil { + return false + } + return bytes.Equal(left, right) +} + +// countWriter is not writing any bytes. It just counts the bytes that would be +// written. +type countWriter struct { + bytes int +} + +// Write implements io.Writer. +func (w *countWriter) Write(p []byte) (int, error) { + w.bytes += len(p) + return len(p), nil +} + +// appendWriter appends to a slice. +type appendWriter struct { + slice []byte +} + +func (w *appendWriter) Write(p []byte) (int, error) { + w.slice = append(w.slice, p...) + return len(p), nil +} diff --git a/pkg/querier/plan/plan_test.go b/pkg/querier/plan/plan_test.go new file mode 100644 index 0000000000000..60f7d3fad1806 --- /dev/null +++ b/pkg/querier/plan/plan_test.go @@ -0,0 +1,26 @@ +package plan + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/pkg/logql/syntax" +) + +func TestMarshalTo(t *testing.T) { + plan := QueryPlan{ + AST: syntax.MustParseExpr(`sum by (foo) (bytes_over_time({app="loki"} [1m]))`), + } + + data := make([]byte, plan.Size()) + _, err := plan.MarshalTo(data) + require.NoError(t, err) + + var buf bytes.Buffer + err = syntax.EncodeJSON(plan.AST, &buf) + require.NoError(t, err) + + require.JSONEq(t, buf.String(), string(data)) +} diff --git a/pkg/querier/queryrange/codec.go b/pkg/querier/queryrange/codec.go index 2167a5134b602..b0c56a7439195 100644 --- a/pkg/querier/queryrange/codec.go +++ b/pkg/querier/queryrange/codec.go @@ -29,6 +29,7 @@ import ( "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" indexStats "github.com/grafana/loki/pkg/storage/stores/index/stats" "github.com/grafana/loki/pkg/util" @@ -259,6 +260,11 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + parsed, err := syntax.ParseExpr(rangeQuery.Query) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiRequest{ Query: rangeQuery.Query, Limit: rangeQuery.Limit, @@ -269,12 +275,21 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer Interval: rangeQuery.Interval.Milliseconds(), Path: r.URL.Path, Shards: rangeQuery.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, nil case InstantQueryOp: req, err := loghttp.ParseInstantQuery(r) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + + parsed, err := syntax.ParseExpr(req.Query) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiInstantRequest{ Query: req.Query, Limit: req.Limit, @@ -282,6 +297,9 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer TimeTs: req.Ts.UTC(), Path: r.URL.Path, Shards: req.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, nil case SeriesOp: req, err := loghttp.ParseAndValidateSeriesQuery(r) @@ -409,6 +427,12 @@ func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) if err != nil { return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + + parsed, err := syntax.ParseExpr(req.Query) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiRequest{ Query: req.Query, Limit: req.Limit, @@ -419,12 +443,21 @@ func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) Interval: req.Interval.Milliseconds(), Path: r.Url, Shards: req.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, ctx, nil case InstantQueryOp: req, err := loghttp.ParseInstantQuery(httpReq) if err != nil { return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + + parsed, err := syntax.ParseExpr(req.Query) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiInstantRequest{ Query: req.Query, Limit: req.Limit, @@ -432,6 +465,9 @@ func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) TimeTs: req.Ts.UTC(), Path: r.Url, Shards: req.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, ctx, nil case SeriesOp: req, err := loghttp.ParseAndValidateSeriesQuery(httpReq) @@ -1429,10 +1465,14 @@ type paramsRangeWrapper struct { *LokiRequest } -func (p paramsRangeWrapper) Query() string { +func (p paramsRangeWrapper) QueryString() string { return p.GetQuery() } +func (p paramsRangeWrapper) GetExpression() syntax.Expr { + return p.LokiRequest.Plan.AST +} + func (p paramsRangeWrapper) Start() time.Time { return p.GetStartTs() } @@ -1459,10 +1499,14 @@ type paramsInstantWrapper struct { *LokiInstantRequest } -func (p paramsInstantWrapper) Query() string { +func (p paramsInstantWrapper) QueryString() string { return p.GetQuery() } +func (p paramsInstantWrapper) GetExpression() syntax.Expr { + return p.LokiInstantRequest.Plan.AST +} + func (p paramsInstantWrapper) Start() time.Time { return p.LokiInstantRequest.GetTimeTs() } @@ -1487,10 +1531,14 @@ type paramsSeriesWrapper struct { *LokiSeriesRequest } -func (p paramsSeriesWrapper) Query() string { +func (p paramsSeriesWrapper) QueryString() string { return p.GetQuery() } +func (p paramsSeriesWrapper) GetExpression() syntax.Expr { + return nil +} + func (p paramsSeriesWrapper) Start() time.Time { return p.LokiSeriesRequest.GetStartTs() } @@ -1515,10 +1563,14 @@ type paramsLabelWrapper struct { *LabelRequest } -func (p paramsLabelWrapper) Query() string { +func (p paramsLabelWrapper) QueryString() string { return p.GetQuery() } +func (p paramsLabelWrapper) GetExpression() syntax.Expr { + return nil +} + func (p paramsLabelWrapper) Start() time.Time { return p.LabelRequest.GetStartTs() } @@ -1543,10 +1595,14 @@ type paramsStatsWrapper struct { *logproto.IndexStatsRequest } -func (p paramsStatsWrapper) Query() string { +func (p paramsStatsWrapper) QueryString() string { return p.GetQuery() } +func (p paramsStatsWrapper) GetExpression() syntax.Expr { + return nil +} + func (p paramsStatsWrapper) Start() time.Time { return p.From.Time() } diff --git a/pkg/querier/queryrange/codec_test.go b/pkg/querier/queryrange/codec_test.go index 77df78aca6958..c722b12af78d8 100644 --- a/pkg/querier/queryrange/codec_test.go +++ b/pkg/querier/queryrange/codec_test.go @@ -25,8 +25,10 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/httpreq" @@ -63,6 +65,9 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { Path: "/query_range", StartTs: start, EndTs: end, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"}`), + }, }, false}, {"query_range", func() (*http.Request, error) { return http.NewRequest(http.MethodGet, @@ -76,6 +81,9 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { Path: "/query_range", StartTs: start, EndTs: end, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"}`), + }, }, false}, {"legacy query_range with refexp", func() (*http.Request, error) { return http.NewRequest(http.MethodGet, @@ -89,6 +97,9 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { Path: "/api/prom/query", StartTs: start, EndTs: end, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"} |~ "foo"`), + }, }, false}, {"series", func() (*http.Request, error) { return http.NewRequest(http.MethodGet, @@ -559,7 +570,13 @@ func Test_codec_DecodeProtobufResponseParity(t *testing.T) { } codec := RequestProtobufCodec{} for i, queryTest := range queryTests { - u := &url.URL{Path: "/loki/api/v1/query_range"} + params := url.Values{ + "query": []string{`{app="foo"}`}, + } + u := &url.URL{ + Path: "/loki/api/v1/query_range", + RawQuery: params.Encode(), + } httpReq := &http.Request{ Method: "GET", RequestURI: u.String(), diff --git a/pkg/querier/queryrange/downstreamer.go b/pkg/querier/queryrange/downstreamer.go index b7a3d2f57a3ff..860aa980fb30b 100644 --- a/pkg/querier/queryrange/downstreamer.go +++ b/pkg/querier/queryrange/downstreamer.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/metadata" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" "github.com/grafana/loki/pkg/util/spanlogger" @@ -34,19 +35,22 @@ type DownstreamHandler struct { next queryrangebase.Handler } -func ParamsToLokiRequest(params logql.Params, shards logql.Shards) queryrangebase.Request { +func ParamsToLokiRequest(params logql.Params) queryrangebase.Request { if logql.GetRangeType(params) == logql.InstantType { return &LokiInstantRequest{ - Query: params.Query(), + Query: params.QueryString(), Limit: params.Limit(), TimeTs: params.Start(), Direction: params.Direction(), Path: "/loki/api/v1/query", // TODO(owen-d): make this derivable - Shards: shards.Encode(), + Shards: params.Shards(), + Plan: &plan.QueryPlan{ + AST: params.GetExpression(), + }, } } return &LokiRequest{ - Query: params.Query(), + Query: params.QueryString(), Limit: params.Limit(), Step: params.Step().Milliseconds(), Interval: params.Interval().Milliseconds(), @@ -54,7 +58,10 @@ func ParamsToLokiRequest(params logql.Params, shards logql.Shards) queryrangebas EndTs: params.End(), Direction: params.Direction(), Path: "/loki/api/v1/query_range", // TODO(owen-d): make this derivable - Shards: shards.Encode(), + Shards: params.Shards(), + Plan: &plan.QueryPlan{ + AST: params.GetExpression(), + }, } } @@ -97,12 +104,12 @@ type instance struct { func (in instance) Downstream(ctx context.Context, queries []logql.DownstreamQuery) ([]logqlmodel.Result, error) { return in.For(ctx, queries, func(qry logql.DownstreamQuery) (logqlmodel.Result, error) { - req := ParamsToLokiRequest(qry.Params, qry.Shards).WithQuery(qry.Expr.String()) + req := ParamsToLokiRequest(qry.Params).WithQuery(qry.Params.GetExpression().String()) sp, ctx := opentracing.StartSpanFromContext(ctx, "DownstreamHandler.instance") defer sp.Finish() logger := spanlogger.FromContext(ctx) defer logger.Finish() - level.Debug(logger).Log("shards", fmt.Sprintf("%+v", qry.Shards), "query", req.GetQuery(), "step", req.GetStep(), "handler", reflect.TypeOf(in.handler)) + level.Debug(logger).Log("shards", fmt.Sprintf("%+v", qry.Params.Shards()), "query", req.GetQuery(), "step", req.GetStep(), "handler", reflect.TypeOf(in.handler)) res, err := in.handler.Do(ctx, req) if err != nil { diff --git a/pkg/querier/queryrange/downstreamer_test.go b/pkg/querier/queryrange/downstreamer_test.go index 552c0c53aa056..e453f03d9a3ee 100644 --- a/pkg/querier/queryrange/downstreamer_test.go +++ b/pkg/querier/queryrange/downstreamer_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "strconv" + "strings" "sync" "testing" "time" @@ -223,8 +225,8 @@ func TestInstanceFor(t *testing.T) { } in := mkIn() newParams := func() logql.Params { - return logql.NewLiteralParams( - "", + params, err := logql.NewLiteralParams( + `{app="foo"}`, time.Now(), time.Now(), 0, @@ -233,6 +235,8 @@ func TestInstanceFor(t *testing.T) { 1000, nil, ) + require.NoError(t, err) + return params } var queries []logql.DownstreamQuery @@ -280,22 +284,32 @@ func TestInstanceFor(t *testing.T) { context.TODO(), []logql.DownstreamQuery{ { - Params: newParams(), - Shards: logql.Shards{ - {Shard: 0, Of: 2}, + Params: logql.ParamsWithShardsOverride{ + Params: newParams(), + ShardsOverride: logql.Shards{ + {Shard: 0, Of: 2}, + }.Encode(), }, }, { - Params: newParams(), - Shards: logql.Shards{ - {Shard: 1, Of: 2}, + Params: logql.ParamsWithShardsOverride{ + Params: newParams(), + ShardsOverride: logql.Shards{ + {Shard: 1, Of: 2}, + }.Encode(), }, }, }, func(qry logql.DownstreamQuery) (logqlmodel.Result, error) { + // Decode shard + s := strings.Split(qry.Params.Shards()[0], "_") + shard, err := strconv.Atoi(s[0]) + if err != nil { + return logqlmodel.Result{}, err + } return logqlmodel.Result{ Data: promql.Scalar{ - V: float64(qry.Shards[0].Shard), + V: float64(shard), }, }, nil }, @@ -309,8 +323,8 @@ func TestInstanceFor(t *testing.T) { } func TestInstanceDownstream(t *testing.T) { - params := logql.NewLiteralParams( - "", + params, err := logql.NewLiteralParams( + `{foo="bar"}`, time.Now(), time.Now(), 0, @@ -319,8 +333,9 @@ func TestInstanceDownstream(t *testing.T) { 1000, nil, ) + require.NoError(t, err) expr, err := syntax.ParseExpr(`{foo="bar"}`) - require.Nil(t, err) + require.NoError(t, err) expectedResp := func() *LokiResponse { return &LokiResponse{ @@ -340,9 +355,10 @@ func TestInstanceDownstream(t *testing.T) { queries := []logql.DownstreamQuery{ { - Expr: expr, - Params: params, - Shards: logql.Shards{{Shard: 0, Of: 2}}, + Params: logql.ParamsWithShardsOverride{ + Params: logql.ParamsWithExpressionOverride{Params: params, ExpressionOverride: expr}, + ShardsOverride: logql.Shards{{Shard: 0, Of: 2}}.Encode(), + }, }, } @@ -353,7 +369,7 @@ func TestInstanceDownstream(t *testing.T) { // for some reason these seemingly can't be checked in their own goroutines, // so we assign them to scoped variables for later comparison. got = req - want = ParamsToLokiRequest(params, queries[0].Shards).WithQuery(expr.String()) + want = ParamsToLokiRequest(queries[0].Params).WithQuery(expr.String()) return expectedResp(), nil }, @@ -484,9 +500,10 @@ func TestDownstreamAccumulatorSimple(t *testing.T) { x = append(x, *s) } // dummy params. Only need to populate direction & limit - params := logql.NewLiteralParams( - "", time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, + params, err := logql.NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, ) + require.NoError(t, err) acc := newDownstreamAccumulator(params, 1) result := logqlmodel.Result{ @@ -542,9 +559,10 @@ func TestDownstreamAccumulatorMultiMerge(t *testing.T) { } // dummy params. Only need to populate direction & limit - params := logql.NewLiteralParams( - "", time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, + params, err := logql.NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, ) + require.NoError(t, err) acc := newDownstreamAccumulator(params, 1) for i := 0; i < nQueries; i++ { diff --git a/pkg/querier/queryrange/marshal.go b/pkg/querier/queryrange/marshal.go index 8b61facf39e4e..0e72b15140e38 100644 --- a/pkg/querier/queryrange/marshal.go +++ b/pkg/querier/queryrange/marshal.go @@ -6,10 +6,12 @@ import ( "context" "fmt" "io" + "net/http" "time" "github.com/gogo/googleapis/google/rpc" "github.com/gogo/status" + "github.com/grafana/dskit/httpgrpc" "github.com/grafana/dskit/user" "github.com/opentracing/opentracing-go" "github.com/prometheus/prometheus/promql" @@ -19,7 +21,9 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/logql/sketch" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/util/httpreq" "github.com/grafana/loki/pkg/util/querylimits" @@ -277,12 +281,32 @@ func (Codec) QueryRequestUnwrap(ctx context.Context, req *QueryRequest) (queryra case *QueryRequest_Series: return concrete.Series, ctx, nil case *QueryRequest_Instant: + if concrete.Instant.Plan == nil { + parsed, err := syntax.ParseExpr(concrete.Instant.GetQuery()) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + concrete.Instant.Plan = &plan.QueryPlan{ + AST: parsed, + } + } + return concrete.Instant, ctx, nil case *QueryRequest_Stats: return concrete.Stats, ctx, nil case *QueryRequest_Volume: return concrete.Volume, ctx, nil case *QueryRequest_Streams: + if concrete.Streams.Plan == nil { + parsed, err := syntax.ParseExpr(concrete.Streams.GetQuery()) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + concrete.Streams.Plan = &plan.QueryPlan{ + AST: parsed, + } + } + return concrete.Streams, ctx, nil case *QueryRequest_Labels: return &LabelRequest{ diff --git a/pkg/querier/queryrange/queryrange.pb.go b/pkg/querier/queryrange/queryrange.pb.go index f3955da153bf9..c2cce1dc514fd 100644 --- a/pkg/querier/queryrange/queryrange.pb.go +++ b/pkg/querier/queryrange/queryrange.pb.go @@ -16,6 +16,7 @@ import ( stats "github.com/grafana/loki/pkg/logqlmodel/stats" _ "github.com/grafana/loki/pkg/push" github_com_grafana_loki_pkg_push "github.com/grafana/loki/pkg/push" + github_com_grafana_loki_pkg_querier_plan "github.com/grafana/loki/pkg/querier/plan" queryrangebase "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" _ "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" github_com_grafana_loki_pkg_querier_queryrange_queryrangebase_definitions "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" @@ -40,15 +41,16 @@ var _ = time.Kitchen const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type LokiRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` - Step int64 `protobuf:"varint,3,opt,name=step,proto3" json:"step,omitempty"` - Interval int64 `protobuf:"varint,9,opt,name=interval,proto3" json:"interval,omitempty"` - StartTs time.Time `protobuf:"bytes,4,opt,name=startTs,proto3,stdtime" json:"startTs"` - EndTs time.Time `protobuf:"bytes,5,opt,name=endTs,proto3,stdtime" json:"endTs"` - Direction logproto.Direction `protobuf:"varint,6,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` - Path string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"` - Shards []string `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + Step int64 `protobuf:"varint,3,opt,name=step,proto3" json:"step,omitempty"` + Interval int64 `protobuf:"varint,9,opt,name=interval,proto3" json:"interval,omitempty"` + StartTs time.Time `protobuf:"bytes,4,opt,name=startTs,proto3,stdtime" json:"startTs"` + EndTs time.Time `protobuf:"bytes,5,opt,name=endTs,proto3,stdtime" json:"endTs"` + Direction logproto.Direction `protobuf:"varint,6,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` + Path string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"` + Shards []string `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards"` + Plan *github_com_grafana_loki_pkg_querier_plan.QueryPlan `protobuf:"bytes,10,opt,name=plan,proto3,customtype=github.com/grafana/loki/pkg/querier/plan.QueryPlan" json:"plan,omitempty"` } func (m *LokiRequest) Reset() { *m = LokiRequest{} } @@ -147,12 +149,13 @@ func (m *LokiRequest) GetShards() []string { } type LokiInstantRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` - TimeTs time.Time `protobuf:"bytes,3,opt,name=timeTs,proto3,stdtime" json:"timeTs"` - Direction logproto.Direction `protobuf:"varint,4,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` - Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` - Shards []string `protobuf:"bytes,6,rep,name=shards,proto3" json:"shards"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + TimeTs time.Time `protobuf:"bytes,3,opt,name=timeTs,proto3,stdtime" json:"timeTs"` + Direction logproto.Direction `protobuf:"varint,4,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` + Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` + Shards []string `protobuf:"bytes,6,rep,name=shards,proto3" json:"shards"` + Plan *github_com_grafana_loki_pkg_querier_plan.QueryPlan `protobuf:"bytes,7,opt,name=plan,proto3,customtype=github.com/grafana/loki/pkg/querier/plan.QueryPlan" json:"plan,omitempty"` } func (m *LokiInstantRequest) Reset() { *m = LokiInstantRequest{} } @@ -1123,99 +1126,101 @@ func init() { } var fileDescriptor_51b9d53b40d11902 = []byte{ - // 1458 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x6f, 0x1b, 0x45, - 0x1b, 0xf7, 0xfa, 0x33, 0x9e, 0x34, 0x79, 0xfb, 0x4e, 0xa2, 0x74, 0xdf, 0xb4, 0xef, 0xae, 0x65, - 0x89, 0xd6, 0x20, 0x58, 0x53, 0xa7, 0xf4, 0x13, 0x10, 0x5d, 0xda, 0xca, 0x15, 0x2d, 0x6a, 0x37, - 0x11, 0x07, 0x6e, 0x13, 0x7b, 0x62, 0x2f, 0xf6, 0x7a, 0x37, 0x3b, 0xe3, 0xa8, 0xb9, 0xf1, 0x07, - 0x80, 0xd4, 0xbf, 0x02, 0x21, 0x51, 0x21, 0x71, 0xe5, 0xc8, 0xa9, 0xc7, 0x1c, 0xab, 0x48, 0x18, - 0xea, 0x72, 0x80, 0x9c, 0xfa, 0x27, 0xa0, 0xf9, 0xd8, 0xf5, 0xac, 0xed, 0xb4, 0x76, 0xb9, 0xb4, - 0x12, 0x97, 0x64, 0x3e, 0x9e, 0xdf, 0xec, 0x3c, 0xbf, 0xe7, 0xf7, 0x3c, 0x33, 0x63, 0x70, 0x2e, - 0xe8, 0xb4, 0xaa, 0xbb, 0x7d, 0x1c, 0xba, 0x38, 0xe4, 0xff, 0xf7, 0x43, 0xd4, 0x6b, 0x61, 0xa5, - 0x69, 0x05, 0xa1, 0x4f, 0x7d, 0x08, 0x46, 0x23, 0xeb, 0xb5, 0x96, 0x4b, 0xdb, 0xfd, 0x6d, 0xab, - 0xe1, 0x7b, 0xd5, 0x96, 0xdf, 0xf2, 0xab, 0x2d, 0xdf, 0x6f, 0x75, 0x31, 0x0a, 0x5c, 0x22, 0x9b, - 0xd5, 0x30, 0x68, 0x54, 0x09, 0x45, 0xb4, 0x4f, 0x04, 0x7e, 0x7d, 0x95, 0x19, 0xf2, 0x26, 0x87, - 0xc8, 0x51, 0x53, 0x9a, 0xf3, 0xde, 0x76, 0x7f, 0xa7, 0x4a, 0x5d, 0x0f, 0x13, 0x8a, 0xbc, 0x40, - 0x1a, 0x9c, 0x66, 0xfb, 0xeb, 0xfa, 0x2d, 0x81, 0x8c, 0x1a, 0x72, 0xf2, 0x7f, 0x89, 0x49, 0xd2, - 0xc1, 0xb4, 0xd1, 0x96, 0x53, 0x25, 0x39, 0xb5, 0xdb, 0xf5, 0xfc, 0x26, 0xee, 0xf2, 0xbd, 0x10, - 0xf1, 0x57, 0x5a, 0xac, 0x30, 0x8b, 0xa0, 0x4f, 0xda, 0xfc, 0x8f, 0x1c, 0xfc, 0xf4, 0xa5, 0x74, - 0x6c, 0x23, 0x82, 0xab, 0x4d, 0xbc, 0xe3, 0xf6, 0x5c, 0xea, 0xfa, 0x3d, 0xa2, 0xb6, 0xe5, 0x22, - 0x17, 0x67, 0x5b, 0x64, 0x9c, 0xe2, 0xf2, 0x41, 0x1a, 0x2c, 0xde, 0xf1, 0x3b, 0xae, 0x83, 0x77, - 0xfb, 0x98, 0x50, 0xb8, 0x0a, 0x72, 0xdc, 0x46, 0xd7, 0x4a, 0x5a, 0xa5, 0xe8, 0x88, 0x0e, 0x1b, - 0xed, 0xba, 0x9e, 0x4b, 0xf5, 0x74, 0x49, 0xab, 0x2c, 0x39, 0xa2, 0x03, 0x21, 0xc8, 0x12, 0x8a, - 0x03, 0x3d, 0x53, 0xd2, 0x2a, 0x19, 0x87, 0xb7, 0xe1, 0x3a, 0x58, 0x70, 0x7b, 0x14, 0x87, 0x7b, - 0xa8, 0xab, 0x17, 0xf9, 0x78, 0xdc, 0x87, 0x1f, 0x83, 0x02, 0xa1, 0x28, 0xa4, 0x5b, 0x44, 0xcf, - 0x96, 0xb4, 0xca, 0x62, 0x6d, 0xdd, 0x12, 0xa1, 0xb0, 0xa2, 0x50, 0x58, 0x5b, 0x51, 0x28, 0xec, - 0x85, 0xc7, 0x03, 0x33, 0xf5, 0xf0, 0x37, 0x53, 0x73, 0x22, 0x10, 0xbc, 0x0a, 0x72, 0xb8, 0xd7, - 0xdc, 0x22, 0x7a, 0x6e, 0x0e, 0xb4, 0x80, 0xc0, 0xf3, 0xa0, 0xd8, 0x74, 0x43, 0xdc, 0x60, 0x9c, - 0xe9, 0xf9, 0x92, 0x56, 0x59, 0xae, 0xad, 0x58, 0x71, 0x68, 0x6f, 0x44, 0x53, 0xce, 0xc8, 0x8a, - 0xb9, 0x17, 0x20, 0xda, 0xd6, 0x0b, 0x9c, 0x09, 0xde, 0x86, 0x65, 0x90, 0x27, 0x6d, 0x14, 0x36, - 0x89, 0xbe, 0x50, 0xca, 0x54, 0x8a, 0x36, 0x38, 0x1a, 0x98, 0x72, 0xc4, 0x91, 0xff, 0xcb, 0x7f, - 0x69, 0x00, 0x32, 0x4a, 0x6f, 0xf7, 0x08, 0x45, 0x3d, 0xfa, 0x2a, 0xcc, 0x7e, 0x08, 0xf2, 0x4c, - 0x94, 0x5b, 0x84, 0x73, 0x3b, 0xab, 0xab, 0x12, 0x93, 0xf4, 0x35, 0x3b, 0x97, 0xaf, 0xb9, 0xa9, - 0xbe, 0xe6, 0x8f, 0xf5, 0xf5, 0x87, 0x2c, 0x38, 0x21, 0xe4, 0x43, 0x02, 0xbf, 0x47, 0x30, 0x03, - 0x6d, 0xf2, 0x14, 0x14, 0x6e, 0x4a, 0x10, 0x1f, 0x71, 0xe4, 0x0c, 0xfc, 0x04, 0x64, 0x6f, 0x20, - 0x8a, 0xb8, 0xcb, 0x8b, 0xb5, 0x55, 0x4b, 0x11, 0x25, 0x5b, 0x8b, 0xcd, 0xd9, 0x6b, 0xcc, 0xab, - 0xa3, 0x81, 0xb9, 0xdc, 0x44, 0x14, 0xbd, 0xeb, 0x7b, 0x2e, 0xc5, 0x5e, 0x40, 0xf7, 0x1d, 0x8e, - 0x84, 0x1f, 0x80, 0xe2, 0xcd, 0x30, 0xf4, 0xc3, 0xad, 0xfd, 0x00, 0x73, 0x8a, 0x8a, 0xf6, 0xa9, - 0xa3, 0x81, 0xb9, 0x82, 0xa3, 0x41, 0x05, 0x31, 0xb2, 0x84, 0x6f, 0x83, 0x1c, 0xef, 0x70, 0x52, - 0x8a, 0xf6, 0xca, 0xd1, 0xc0, 0xfc, 0x0f, 0x87, 0x28, 0xe6, 0xc2, 0x22, 0xc9, 0x61, 0x6e, 0x26, - 0x0e, 0xe3, 0x50, 0xe6, 0xd5, 0x50, 0xea, 0xa0, 0xb0, 0x87, 0x43, 0xc2, 0x96, 0x29, 0xf0, 0xf1, - 0xa8, 0x0b, 0xaf, 0x03, 0xc0, 0x88, 0x71, 0x09, 0x75, 0x1b, 0x4c, 0x4f, 0x8c, 0x8c, 0x25, 0x4b, - 0x94, 0x0b, 0x07, 0x93, 0x7e, 0x97, 0xda, 0x50, 0xb2, 0xa0, 0x18, 0x3a, 0x4a, 0x1b, 0x3e, 0xd2, - 0x40, 0xa1, 0x8e, 0x51, 0x13, 0x87, 0x44, 0x2f, 0x96, 0x32, 0x95, 0xc5, 0xda, 0x5b, 0x96, 0x5a, - 0x1b, 0xee, 0x85, 0xbe, 0x87, 0x69, 0x1b, 0xf7, 0x49, 0x14, 0x20, 0x61, 0x6d, 0x77, 0x0e, 0x07, - 0xe6, 0xb6, 0x5a, 0x51, 0x43, 0xb4, 0x83, 0x7a, 0xa8, 0xda, 0xf5, 0x3b, 0x6e, 0x75, 0xee, 0x7a, - 0x74, 0xec, 0x77, 0x8e, 0x06, 0xa6, 0xf6, 0x9e, 0x13, 0x6d, 0xb1, 0xfc, 0xab, 0x06, 0xfe, 0xcb, - 0x22, 0xbc, 0xc9, 0xd6, 0x26, 0x4a, 0x62, 0x78, 0x88, 0x36, 0xda, 0xba, 0xc6, 0x64, 0xe6, 0x88, - 0x8e, 0x5a, 0x2c, 0xd2, 0xff, 0xa8, 0x58, 0x64, 0xe6, 0x2f, 0x16, 0x51, 0x36, 0x64, 0xa7, 0x66, - 0x43, 0xee, 0xd8, 0x6c, 0xf8, 0x26, 0x23, 0x32, 0x3f, 0xf2, 0x6f, 0x8e, 0x9c, 0xb8, 0x15, 0xe7, - 0x44, 0x86, 0xef, 0x36, 0x96, 0x9a, 0x58, 0xeb, 0x76, 0x13, 0xf7, 0xa8, 0xbb, 0xe3, 0xe2, 0xf0, - 0x25, 0x99, 0xa1, 0xc8, 0x2d, 0x93, 0x94, 0x9b, 0xaa, 0x95, 0xec, 0x6b, 0xaf, 0x95, 0xb1, 0xec, - 0xc8, 0xbd, 0x42, 0x76, 0x94, 0x9f, 0xa7, 0xc1, 0x1a, 0x0b, 0xc7, 0x1d, 0xb4, 0x8d, 0xbb, 0x9f, - 0x23, 0x6f, 0xce, 0x90, 0x9c, 0x55, 0x42, 0x52, 0xb4, 0xe1, 0xbf, 0x94, 0xcf, 0x40, 0xf9, 0x77, - 0x1a, 0x58, 0x88, 0x6a, 0x38, 0xb4, 0x00, 0x10, 0x30, 0x5e, 0xa6, 0x05, 0xd1, 0xcb, 0x0c, 0x1c, - 0xc6, 0xa3, 0x8e, 0x62, 0x01, 0xbf, 0x02, 0x79, 0xd1, 0x93, 0x59, 0x70, 0x4a, 0xc9, 0x02, 0x1a, - 0x62, 0xe4, 0x5d, 0x6f, 0xa2, 0x80, 0xe2, 0xd0, 0xbe, 0xc2, 0x76, 0x71, 0x38, 0x30, 0xcf, 0xbd, - 0x88, 0x22, 0x7e, 0xc3, 0x12, 0x38, 0x16, 0x5c, 0xf1, 0x4d, 0x47, 0x7e, 0xa1, 0xfc, 0xad, 0x06, - 0x4e, 0xb2, 0x8d, 0x32, 0x6a, 0x62, 0x55, 0xdc, 0x00, 0x0b, 0xa1, 0x6c, 0xf3, 0xed, 0x2e, 0xd6, - 0xca, 0x56, 0x92, 0xd6, 0x29, 0x54, 0xda, 0xd9, 0xc7, 0x03, 0x53, 0x73, 0x62, 0x24, 0xdc, 0x48, - 0xd0, 0x98, 0x9e, 0x46, 0x23, 0x83, 0xa4, 0x12, 0xc4, 0xfd, 0x9c, 0x06, 0xf0, 0x76, 0xaf, 0x89, - 0x1f, 0x30, 0xf1, 0x8d, 0x74, 0xda, 0x9f, 0xd8, 0xd1, 0x99, 0x11, 0x29, 0x93, 0xf6, 0xf6, 0xb5, - 0xc3, 0x81, 0x79, 0xe9, 0x45, 0xac, 0xbc, 0x00, 0xac, 0xb8, 0xa0, 0x0a, 0x37, 0xfd, 0xfa, 0x9f, - 0x2b, 0x3f, 0xa6, 0xc1, 0xf2, 0x17, 0x7e, 0xb7, 0xef, 0xe1, 0x98, 0x38, 0x6f, 0x82, 0x38, 0x7d, - 0x44, 0x5c, 0xd2, 0xd6, 0xbe, 0x74, 0x38, 0x30, 0x37, 0x66, 0x22, 0x2d, 0x09, 0x7c, 0x73, 0x09, - 0x7b, 0x94, 0x06, 0xab, 0x5b, 0x7e, 0xf0, 0xd9, 0x26, 0x7f, 0xbe, 0x28, 0x75, 0x11, 0x4f, 0xd0, - 0xb6, 0x3a, 0xa2, 0x8d, 0x21, 0xee, 0x22, 0x1a, 0xba, 0x0f, 0xec, 0x8d, 0xc3, 0x81, 0x59, 0x9d, - 0x89, 0xb2, 0x11, 0xe8, 0xcd, 0xa5, 0xeb, 0x97, 0x34, 0x58, 0xbb, 0xdf, 0x47, 0x3d, 0xea, 0x76, - 0xb1, 0xa0, 0x2c, 0x26, 0x6c, 0x7f, 0x82, 0x30, 0x63, 0x44, 0x58, 0x12, 0x23, 0xa9, 0xfb, 0xe8, - 0x70, 0x60, 0x5e, 0x99, 0x89, 0xba, 0x69, 0xf0, 0x37, 0x97, 0xc4, 0x9f, 0xb2, 0x60, 0xe9, 0x3e, - 0x5b, 0x25, 0xe6, 0xee, 0x1d, 0x20, 0x8f, 0x5c, 0xc9, 0x1c, 0x8c, 0xee, 0x68, 0x61, 0xd0, 0xb0, - 0x36, 0xe5, 0x61, 0x2c, 0x2c, 0xe0, 0x65, 0x90, 0x27, 0xfc, 0x26, 0x24, 0x0b, 0xaa, 0x31, 0xfe, - 0x6a, 0x48, 0xde, 0xb9, 0xea, 0x29, 0x47, 0xda, 0xb3, 0xb7, 0x54, 0x97, 0x5d, 0x00, 0xa2, 0x9b, - 0x60, 0x79, 0x1c, 0x39, 0x79, 0x3d, 0x60, 0x68, 0x81, 0x81, 0x17, 0x41, 0x8e, 0x57, 0x6e, 0xf9, - 0x62, 0x4d, 0x7c, 0x76, 0xb2, 0x84, 0xd6, 0x53, 0x8e, 0x30, 0x87, 0x35, 0x90, 0x0d, 0x42, 0xdf, - 0x93, 0xa7, 0xe8, 0x99, 0xf1, 0x6f, 0xaa, 0xc7, 0x4e, 0x3d, 0xe5, 0x70, 0x5b, 0x78, 0x81, 0x5d, - 0x79, 0xd9, 0x79, 0x45, 0xf8, 0x13, 0x82, 0x95, 0xac, 0x31, 0x98, 0x02, 0x89, 0x4c, 0xe1, 0x05, - 0x90, 0xdf, 0xe3, 0x65, 0x89, 0xbf, 0x2f, 0xd8, 0xdd, 0x51, 0x01, 0x25, 0x0b, 0x16, 0xf3, 0x4b, - 0xd8, 0xc2, 0x5b, 0xe0, 0x04, 0xf5, 0x83, 0x4e, 0x54, 0x00, 0xe4, 0xf3, 0xa3, 0xa4, 0x62, 0xa7, - 0x15, 0x88, 0x7a, 0xca, 0x49, 0xe0, 0xe0, 0x3d, 0x70, 0x72, 0x37, 0x21, 0x53, 0x4c, 0xf8, 0xbb, - 0x7f, 0x8c, 0xe7, 0xe9, 0xd9, 0x53, 0x4f, 0x39, 0x13, 0x68, 0x1b, 0x8c, 0x32, 0xaa, 0xfc, 0x47, - 0x06, 0x9c, 0x90, 0x9a, 0x11, 0x6f, 0x85, 0x4b, 0xb1, 0x0c, 0x84, 0x64, 0xfe, 0x7f, 0x9c, 0x0c, - 0xb8, 0xb9, 0xa2, 0x82, 0xf7, 0x63, 0x15, 0x08, 0xfd, 0xac, 0x8d, 0xb2, 0x94, 0xc7, 0x5f, 0x41, - 0xc8, 0xc8, 0x6f, 0x44, 0x91, 0x17, 0xb2, 0x39, 0x3d, 0xfd, 0xdc, 0x8d, 0x50, 0x32, 0xec, 0x57, - 0x41, 0xc1, 0x15, 0xcf, 0xfe, 0x69, 0x82, 0x99, 0xfc, 0x55, 0x80, 0x05, 0x52, 0x02, 0xe0, 0xc6, - 0x28, 0xfc, 0x42, 0x35, 0xa7, 0x26, 0xc3, 0x1f, 0x83, 0xa2, 0xe8, 0x9f, 0x8f, 0xa3, 0x9f, 0x97, - 0x98, 0x89, 0xc3, 0x2a, 0x76, 0x4c, 0x86, 0xbe, 0x0e, 0x16, 0x3c, 0x4c, 0x11, 0xbb, 0xcb, 0xea, - 0x05, 0x5e, 0x37, 0xce, 0x26, 0x43, 0x35, 0xe2, 0xdb, 0xba, 0x2b, 0x0d, 0x6f, 0xf6, 0x68, 0xb8, - 0x2f, 0xaf, 0x2d, 0x31, 0x7a, 0xfd, 0x1a, 0x58, 0x4a, 0x18, 0xc0, 0x93, 0x20, 0xd3, 0xc1, 0xd1, - 0x2f, 0x1c, 0xac, 0xc9, 0x1e, 0x77, 0x7b, 0xa8, 0xdb, 0xc7, 0x9c, 0xf6, 0xa2, 0x23, 0x3a, 0x57, - 0xd3, 0x97, 0x35, 0xbb, 0x08, 0x0a, 0xa1, 0xf8, 0x8a, 0xdd, 0x3c, 0x78, 0x6a, 0xa4, 0x9e, 0x3c, - 0x35, 0x52, 0xcf, 0x9f, 0x1a, 0xda, 0xd7, 0x43, 0x43, 0xfb, 0x7e, 0x68, 0x68, 0x8f, 0x87, 0x86, - 0x76, 0x30, 0x34, 0xb4, 0xdf, 0x87, 0x86, 0xf6, 0xe7, 0xd0, 0x48, 0x3d, 0x1f, 0x1a, 0xda, 0xc3, - 0x67, 0x46, 0xea, 0xe0, 0x99, 0x91, 0x7a, 0xf2, 0xcc, 0x48, 0x7d, 0x69, 0xcd, 0x57, 0xc2, 0xb6, - 0xf3, 0x9c, 0x96, 0x8d, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x66, 0x27, 0xc9, 0x7f, 0x7f, 0x14, - 0x00, 0x00, + // 1498 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0x4b, 0x6f, 0xdb, 0xc6, + 0x1a, 0x15, 0xf5, 0xb4, 0xc6, 0x8f, 0x9b, 0x3b, 0x36, 0x1c, 0x5e, 0x27, 0x97, 0x14, 0x04, 0xdc, + 0x44, 0xb7, 0x68, 0xa9, 0xc6, 0x4e, 0xf3, 0x6c, 0x8b, 0x86, 0x4d, 0x02, 0xa7, 0x4d, 0x8a, 0x84, + 0x36, 0xba, 0xe8, 0x6e, 0x2c, 0x8d, 0x25, 0x56, 0x7c, 0x99, 0x33, 0x32, 0xe2, 0x5d, 0x7f, 0x40, + 0x0b, 0xe4, 0x07, 0x74, 0x5d, 0x14, 0x68, 0x50, 0xa0, 0x8b, 0x6e, 0xba, 0xec, 0x2a, 0xcb, 0x2c, + 0x03, 0x01, 0x65, 0x1b, 0xa5, 0x8b, 0xc2, 0xab, 0xfc, 0x84, 0x62, 0x1e, 0xa4, 0x48, 0x49, 0x49, + 0xe4, 0xb4, 0x8b, 0x04, 0xe8, 0x46, 0x9a, 0x19, 0x7e, 0x87, 0x1c, 0x9e, 0x73, 0xbe, 0x6f, 0x66, + 0x08, 0x4e, 0x07, 0xbd, 0x4e, 0x73, 0xaf, 0x8f, 0x43, 0x1b, 0x87, 0xfc, 0xff, 0x20, 0x44, 0x5e, + 0x07, 0xa7, 0x9a, 0x46, 0x10, 0xfa, 0xd4, 0x87, 0x60, 0x34, 0xb2, 0xb6, 0xde, 0xb1, 0x69, 0xb7, + 0xbf, 0x63, 0xb4, 0x7c, 0xb7, 0xd9, 0xf1, 0x3b, 0x7e, 0xb3, 0xe3, 0xfb, 0x1d, 0x07, 0xa3, 0xc0, + 0x26, 0xb2, 0xd9, 0x0c, 0x83, 0x56, 0x93, 0x50, 0x44, 0xfb, 0x44, 0xe0, 0xd7, 0x56, 0x58, 0x20, + 0x6f, 0x72, 0x88, 0x1c, 0xd5, 0x65, 0x38, 0xef, 0xed, 0xf4, 0x77, 0x9b, 0xd4, 0x76, 0x31, 0xa1, + 0xc8, 0x0d, 0x64, 0xc0, 0x09, 0x36, 0x3f, 0xc7, 0xef, 0x08, 0x64, 0xdc, 0x90, 0x17, 0xff, 0x93, + 0xb9, 0x48, 0x7a, 0x98, 0xb6, 0xba, 0xf2, 0x52, 0x4d, 0x5e, 0xda, 0x73, 0x5c, 0xbf, 0x8d, 0x1d, + 0x3e, 0x17, 0x22, 0x7e, 0x65, 0xc4, 0x32, 0x8b, 0x08, 0xfa, 0xa4, 0xcb, 0x7f, 0xe4, 0xe0, 0x87, + 0x2f, 0xa4, 0x63, 0x07, 0x11, 0xdc, 0x6c, 0xe3, 0x5d, 0xdb, 0xb3, 0xa9, 0xed, 0x7b, 0x24, 0xdd, + 0x96, 0x37, 0x39, 0x37, 0xdb, 0x4d, 0xc6, 0x29, 0xae, 0x7f, 0x5d, 0x00, 0xf3, 0x37, 0xfd, 0x9e, + 0x6d, 0xe1, 0xbd, 0x3e, 0x26, 0x14, 0xae, 0x80, 0x12, 0x8f, 0x51, 0x95, 0x9a, 0xd2, 0xa8, 0x5a, + 0xa2, 0xc3, 0x46, 0x1d, 0xdb, 0xb5, 0xa9, 0x9a, 0xaf, 0x29, 0x8d, 0x45, 0x4b, 0x74, 0x20, 0x04, + 0x45, 0x42, 0x71, 0xa0, 0x16, 0x6a, 0x4a, 0xa3, 0x60, 0xf1, 0x36, 0x5c, 0x03, 0x73, 0xb6, 0x47, + 0x71, 0xb8, 0x8f, 0x1c, 0xb5, 0xca, 0xc7, 0x93, 0x3e, 0x7c, 0x1f, 0x54, 0x08, 0x45, 0x21, 0xdd, + 0x26, 0x6a, 0xb1, 0xa6, 0x34, 0xe6, 0xd7, 0xd7, 0x0c, 0x21, 0x85, 0x11, 0x4b, 0x61, 0x6c, 0xc7, + 0x52, 0x98, 0x73, 0x0f, 0x22, 0x3d, 0x77, 0xef, 0x57, 0x5d, 0xb1, 0x62, 0x10, 0xbc, 0x04, 0x4a, + 0xd8, 0x6b, 0x6f, 0x13, 0xb5, 0x74, 0x04, 0xb4, 0x80, 0xc0, 0x33, 0xa0, 0xda, 0xb6, 0x43, 0xdc, + 0x62, 0x9c, 0xa9, 0xe5, 0x9a, 0xd2, 0x58, 0x5a, 0x5f, 0x36, 0x12, 0x69, 0xaf, 0xc6, 0x97, 0xac, + 0x51, 0x14, 0x7b, 0xbd, 0x00, 0xd1, 0xae, 0x5a, 0xe1, 0x4c, 0xf0, 0x36, 0xac, 0x83, 0x32, 0xe9, + 0xa2, 0xb0, 0x4d, 0xd4, 0xb9, 0x5a, 0xa1, 0x51, 0x35, 0xc1, 0x61, 0xa4, 0xcb, 0x11, 0x4b, 0xfe, + 0xc3, 0x8f, 0x40, 0x31, 0x70, 0x90, 0xa7, 0x82, 0x9a, 0xd2, 0x58, 0x30, 0xcf, 0x0d, 0x22, 0x3d, + 0xe3, 0xdd, 0x10, 0xed, 0x22, 0x0f, 0x35, 0x1d, 0xbf, 0x67, 0x37, 0xd3, 0xa2, 0x31, 0x8c, 0x71, + 0x87, 0xd1, 0x7d, 0xdb, 0x41, 0x9e, 0xc5, 0xef, 0x51, 0xff, 0x31, 0x0f, 0x20, 0x93, 0xe7, 0x86, + 0x47, 0x28, 0xf2, 0xe8, 0xcb, 0xa8, 0xf4, 0x2e, 0x28, 0x33, 0x83, 0x6f, 0x13, 0xae, 0xd3, 0xac, + 0xb4, 0x49, 0x4c, 0x96, 0xb7, 0xe2, 0x91, 0x78, 0x2b, 0x4d, 0xe5, 0xad, 0xfc, 0x42, 0xde, 0x2a, + 0x7f, 0x03, 0x6f, 0xdf, 0x15, 0xc1, 0x82, 0xb0, 0x35, 0x09, 0x7c, 0x8f, 0x60, 0x36, 0x81, 0x2d, + 0x5e, 0x1a, 0x04, 0x65, 0x72, 0x02, 0x7c, 0xc4, 0x92, 0x57, 0xe0, 0x07, 0xa0, 0x78, 0x15, 0x51, + 0xc4, 0xe9, 0x9b, 0x5f, 0x5f, 0x31, 0x52, 0xc9, 0xc2, 0xee, 0xc5, 0xae, 0x99, 0xab, 0x8c, 0xa1, + 0xc3, 0x48, 0x5f, 0x6a, 0x23, 0x8a, 0xde, 0xf4, 0x5d, 0x9b, 0x62, 0x37, 0xa0, 0x07, 0x16, 0x47, + 0xc2, 0x77, 0x40, 0xf5, 0x5a, 0x18, 0xfa, 0xe1, 0xf6, 0x41, 0x80, 0x39, 0xdd, 0x55, 0xf3, 0xf8, + 0x61, 0xa4, 0x2f, 0xe3, 0x78, 0x30, 0x85, 0x18, 0x45, 0xc2, 0xff, 0x83, 0x12, 0xef, 0x70, 0x82, + 0xab, 0xe6, 0xf2, 0x61, 0xa4, 0xff, 0x8b, 0x43, 0x52, 0xe1, 0x22, 0x22, 0xab, 0x47, 0x69, 0x26, + 0x3d, 0x12, 0x5b, 0x94, 0xd3, 0xb6, 0x50, 0x41, 0x65, 0x1f, 0x87, 0x84, 0xdd, 0xa6, 0xc2, 0xc7, + 0xe3, 0x2e, 0xbc, 0x02, 0x00, 0x23, 0xc6, 0x26, 0xd4, 0x6e, 0x31, 0x9f, 0x33, 0x32, 0x16, 0x0d, + 0x51, 0xc6, 0x2c, 0x4c, 0xfa, 0x0e, 0x35, 0xa1, 0x64, 0x21, 0x15, 0x68, 0xa5, 0xda, 0xf0, 0xbe, + 0x02, 0x2a, 0x9b, 0x18, 0xb5, 0x71, 0x48, 0xd4, 0x6a, 0xad, 0xd0, 0x98, 0x5f, 0xff, 0x9f, 0x91, + 0xae, 0x59, 0xb7, 0x43, 0xdf, 0xc5, 0xb4, 0x8b, 0xfb, 0x24, 0x16, 0x48, 0x44, 0x9b, 0xbd, 0x41, + 0xa4, 0xef, 0xcc, 0xa2, 0xfa, 0x4c, 0x75, 0xf2, 0x99, 0xcf, 0x39, 0x8c, 0x74, 0xe5, 0x2d, 0x2b, + 0x9e, 0x62, 0xfd, 0x17, 0x05, 0xfc, 0x9b, 0x29, 0xbc, 0xc5, 0xee, 0x4d, 0x52, 0x49, 0xe6, 0x22, + 0xda, 0xea, 0xaa, 0x0a, 0xb3, 0xac, 0x25, 0x3a, 0xe9, 0x22, 0x96, 0xff, 0x4b, 0x45, 0xac, 0x70, + 0xf4, 0x22, 0x16, 0x67, 0x56, 0x71, 0x6a, 0x66, 0x95, 0x9e, 0x95, 0x59, 0xf5, 0x2f, 0x0b, 0xa2, + 0x8a, 0xc4, 0xef, 0x77, 0x84, 0x9c, 0xb8, 0x9e, 0xe4, 0x44, 0x81, 0xcf, 0x36, 0xb1, 0x9a, 0xb8, + 0xd7, 0x8d, 0x36, 0xf6, 0xa8, 0xbd, 0x6b, 0xe3, 0xf0, 0x05, 0x99, 0x91, 0xb2, 0x5b, 0x21, 0x6b, + 0xb7, 0xb4, 0x57, 0x8a, 0xaf, 0xbc, 0x57, 0xc6, 0xb2, 0xa3, 0xf4, 0x12, 0xd9, 0x51, 0x7f, 0x9a, + 0x07, 0xab, 0x4c, 0x8e, 0x9b, 0x68, 0x07, 0x3b, 0x9f, 0x20, 0xf7, 0x88, 0x92, 0x9c, 0x4a, 0x49, + 0x52, 0x35, 0xe1, 0x3f, 0x94, 0xcf, 0x40, 0xf9, 0x37, 0x0a, 0x98, 0x8b, 0x6b, 0x38, 0x34, 0x00, + 0x10, 0x30, 0x5e, 0xa6, 0x05, 0xd1, 0x4b, 0x0c, 0x1c, 0x26, 0xa3, 0x56, 0x2a, 0x02, 0x7e, 0x0e, + 0xca, 0xa2, 0x27, 0xb3, 0xe0, 0x78, 0x2a, 0x0b, 0x68, 0x88, 0x91, 0x7b, 0xa5, 0x8d, 0x02, 0x8a, + 0x43, 0xf3, 0x22, 0x9b, 0xc5, 0x20, 0xd2, 0x4f, 0x3f, 0x8f, 0x22, 0xbe, 0xf3, 0x13, 0x38, 0x26, + 0xae, 0x78, 0xa6, 0x25, 0x9f, 0x50, 0xff, 0x4a, 0x01, 0xc7, 0xd8, 0x44, 0x19, 0x35, 0x89, 0x2b, + 0xae, 0x82, 0xb9, 0x50, 0xb6, 0xf9, 0x74, 0xe7, 0xd7, 0xeb, 0x46, 0x96, 0xd6, 0x29, 0x54, 0x9a, + 0xc5, 0x07, 0x91, 0xae, 0x58, 0x09, 0x12, 0x6e, 0x64, 0x68, 0xcc, 0x4f, 0xa3, 0x91, 0x41, 0x72, + 0x19, 0xe2, 0x7e, 0xca, 0x03, 0x78, 0xc3, 0x6b, 0xe3, 0xbb, 0xcc, 0x7c, 0x23, 0x9f, 0xf6, 0x27, + 0x66, 0x74, 0x72, 0x44, 0xca, 0x64, 0xbc, 0x79, 0x79, 0x10, 0xe9, 0xe7, 0x9f, 0xc7, 0xca, 0x73, + 0xc0, 0xa9, 0x57, 0x48, 0x1b, 0x37, 0xff, 0xea, 0xaf, 0x2b, 0xdf, 0xe7, 0xc1, 0xd2, 0xa7, 0xbe, + 0xd3, 0x77, 0x71, 0x42, 0x9c, 0x3b, 0x41, 0x9c, 0x3a, 0x22, 0x2e, 0x1b, 0x6b, 0x9e, 0x1f, 0x44, + 0xfa, 0xc6, 0x4c, 0xa4, 0x65, 0x81, 0xaf, 0x2f, 0x61, 0xf7, 0xf3, 0x60, 0x65, 0xdb, 0x0f, 0x3e, + 0xde, 0xe2, 0xc7, 0xaa, 0x54, 0x5d, 0xc4, 0x13, 0xb4, 0xad, 0x8c, 0x68, 0x63, 0x88, 0x5b, 0x88, + 0x86, 0xf6, 0x5d, 0x73, 0x63, 0x10, 0xe9, 0xcd, 0x99, 0x28, 0x1b, 0x81, 0x5e, 0x5f, 0xba, 0x7e, + 0xce, 0x83, 0xd5, 0x3b, 0x7d, 0xe4, 0x51, 0xdb, 0xc1, 0x82, 0xb2, 0x84, 0xb0, 0x83, 0x09, 0xc2, + 0xb4, 0x11, 0x61, 0x59, 0x8c, 0xa4, 0xee, 0xbd, 0x41, 0xa4, 0x5f, 0x9c, 0x89, 0xba, 0x69, 0xf0, + 0xd7, 0x97, 0xc4, 0x1f, 0x8a, 0x60, 0x91, 0x1f, 0x1f, 0x12, 0xee, 0xde, 0x00, 0x72, 0xc9, 0x95, + 0xcc, 0xc1, 0x78, 0x8f, 0x16, 0x06, 0x2d, 0x63, 0x4b, 0x2e, 0xc6, 0x22, 0x02, 0x5e, 0x00, 0x65, + 0xc2, 0x77, 0x42, 0xb2, 0xa0, 0x6a, 0xe3, 0xa7, 0x86, 0xec, 0x9e, 0x6b, 0x33, 0x67, 0xc9, 0x78, + 0x76, 0x2e, 0x73, 0xd8, 0x06, 0x20, 0xde, 0x09, 0xd6, 0xc7, 0x91, 0x93, 0xdb, 0x03, 0x86, 0x16, + 0x18, 0x78, 0x0e, 0x94, 0x78, 0xe5, 0x96, 0x27, 0xe9, 0xcc, 0x63, 0x27, 0x4b, 0xe8, 0x66, 0xce, + 0x12, 0xe1, 0x70, 0x1d, 0x14, 0x83, 0xd0, 0x77, 0xe5, 0x2a, 0x7a, 0x72, 0xfc, 0x99, 0xe9, 0x65, + 0x67, 0x33, 0x67, 0xf1, 0x58, 0x78, 0x96, 0x6d, 0x79, 0xd9, 0x7a, 0x45, 0xf8, 0x11, 0x82, 0x95, + 0xac, 0x31, 0x58, 0x0a, 0x12, 0x87, 0xc2, 0xb3, 0xa0, 0xbc, 0xcf, 0xcb, 0x12, 0x3f, 0x5f, 0xb0, + 0xbd, 0x63, 0x0a, 0x94, 0x2d, 0x58, 0xec, 0xbd, 0x44, 0x2c, 0xbc, 0x0e, 0x16, 0xa8, 0x1f, 0xf4, + 0xe2, 0x02, 0x20, 0x8f, 0x1f, 0xb5, 0x34, 0x76, 0x5a, 0x81, 0xd8, 0xcc, 0x59, 0x19, 0x1c, 0xbc, + 0x0d, 0x8e, 0xed, 0x65, 0x6c, 0x8a, 0x09, 0xff, 0x1e, 0x31, 0xc6, 0xf3, 0xf4, 0xec, 0xd9, 0xcc, + 0x59, 0x13, 0x68, 0x13, 0x8c, 0x32, 0xaa, 0xfe, 0x7b, 0x01, 0x2c, 0x48, 0xcf, 0x88, 0xb3, 0xc2, + 0xf9, 0xc4, 0x06, 0xc2, 0x32, 0xff, 0x7d, 0x96, 0x0d, 0x78, 0x78, 0xca, 0x05, 0x6f, 0x27, 0x2e, + 0x10, 0xfe, 0x59, 0x1d, 0x65, 0x29, 0xd7, 0x3f, 0x85, 0x90, 0xca, 0x6f, 0xc4, 0xca, 0x0b, 0xdb, + 0x9c, 0x98, 0xbe, 0xee, 0xc6, 0x28, 0x29, 0xfb, 0x25, 0x50, 0xb1, 0xc5, 0x27, 0x84, 0x69, 0x86, + 0x99, 0xfc, 0xc2, 0xc0, 0x84, 0x94, 0x00, 0xb8, 0x31, 0x92, 0x5f, 0xb8, 0xe6, 0xf8, 0xa4, 0xfc, + 0x09, 0x28, 0x56, 0xff, 0x4c, 0xa2, 0x7e, 0x59, 0x62, 0x26, 0x16, 0xab, 0xe4, 0xc5, 0xa4, 0xf4, + 0x9b, 0x60, 0xce, 0xc5, 0x14, 0xb1, 0xbd, 0xac, 0x5a, 0xe1, 0x75, 0xe3, 0x54, 0x56, 0xaa, 0x11, + 0xdf, 0xc6, 0x2d, 0x19, 0x78, 0xcd, 0xa3, 0xe1, 0x81, 0xdc, 0xb6, 0x24, 0xe8, 0xb5, 0xcb, 0x60, + 0x31, 0x13, 0x00, 0x8f, 0x81, 0x42, 0x0f, 0xc7, 0x5f, 0x4b, 0x58, 0x93, 0x1d, 0xee, 0xf6, 0x91, + 0xd3, 0xc7, 0x9c, 0xf6, 0xaa, 0x25, 0x3a, 0x97, 0xf2, 0x17, 0x14, 0xb3, 0x0a, 0x2a, 0xa1, 0x78, + 0x8a, 0xd9, 0x7e, 0xf8, 0x58, 0xcb, 0x3d, 0x7a, 0xac, 0xe5, 0x9e, 0x3e, 0xd6, 0x94, 0x2f, 0x86, + 0x9a, 0xf2, 0xed, 0x50, 0x53, 0x1e, 0x0c, 0x35, 0xe5, 0xe1, 0x50, 0x53, 0x7e, 0x1b, 0x6a, 0xca, + 0x1f, 0x43, 0x2d, 0xf7, 0x74, 0xa8, 0x29, 0xf7, 0x9e, 0x68, 0xb9, 0x87, 0x4f, 0xb4, 0xdc, 0xa3, + 0x27, 0x5a, 0xee, 0x33, 0xe3, 0x68, 0x25, 0x6c, 0xa7, 0xcc, 0x69, 0xd9, 0xf8, 0x33, 0x00, 0x00, + 0xff, 0xff, 0x33, 0xb4, 0xee, 0x07, 0x17, 0x15, 0x00, 0x00, } func (this *LokiRequest) Equal(that interface{}) bool { @@ -1269,6 +1274,13 @@ func (this *LokiRequest) Equal(that interface{}) bool { return false } } + if that1.Plan == nil { + if this.Plan != nil { + return false + } + } else if !this.Plan.Equal(*that1.Plan) { + return false + } return true } func (this *LokiInstantRequest) Equal(that interface{}) bool { @@ -1313,6 +1325,13 @@ func (this *LokiInstantRequest) Equal(that interface{}) bool { return false } } + if that1.Plan == nil { + if this.Plan != nil { + return false + } + } else if !this.Plan.Equal(*that1.Plan) { + return false + } return true } func (this *LokiResponse) Equal(that interface{}) bool { @@ -2120,7 +2139,7 @@ func (this *LokiRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 13) + s := make([]string, 0, 14) s = append(s, "&queryrange.LokiRequest{") s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") @@ -2131,6 +2150,7 @@ func (this *LokiRequest) GoString() string { s = append(s, "Direction: "+fmt.Sprintf("%#v", this.Direction)+",\n") s = append(s, "Path: "+fmt.Sprintf("%#v", this.Path)+",\n") s = append(s, "Shards: "+fmt.Sprintf("%#v", this.Shards)+",\n") + s = append(s, "Plan: "+fmt.Sprintf("%#v", this.Plan)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2138,7 +2158,7 @@ func (this *LokiInstantRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 11) s = append(s, "&queryrange.LokiInstantRequest{") s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") @@ -2146,6 +2166,7 @@ func (this *LokiInstantRequest) GoString() string { s = append(s, "Direction: "+fmt.Sprintf("%#v", this.Direction)+",\n") s = append(s, "Path: "+fmt.Sprintf("%#v", this.Path)+",\n") s = append(s, "Shards: "+fmt.Sprintf("%#v", this.Shards)+",\n") + s = append(s, "Plan: "+fmt.Sprintf("%#v", this.Plan)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2463,6 +2484,18 @@ func (m *LokiRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Plan != nil { + { + size := m.Plan.Size() + i -= size + if _, err := m.Plan.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } if m.Interval != 0 { i = encodeVarintQueryrange(dAtA, i, uint64(m.Interval)) i-- @@ -2545,6 +2578,18 @@ func (m *LokiInstantRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Plan != nil { + { + size := m.Plan.Size() + i -= size + if _, err := m.Plan.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if len(m.Shards) > 0 { for iNdEx := len(m.Shards) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Shards[iNdEx]) @@ -3594,6 +3639,10 @@ func (m *LokiRequest) Size() (n int) { if m.Interval != 0 { n += 1 + sovQueryrange(uint64(m.Interval)) } + if m.Plan != nil { + l = m.Plan.Size() + n += 1 + l + sovQueryrange(uint64(l)) + } return n } @@ -3625,6 +3674,10 @@ func (m *LokiInstantRequest) Size() (n int) { n += 1 + l + sovQueryrange(uint64(l)) } } + if m.Plan != nil { + l = m.Plan.Size() + n += 1 + l + sovQueryrange(uint64(l)) + } return n } @@ -4092,6 +4145,7 @@ func (this *LokiRequest) String() string { `Path:` + fmt.Sprintf("%v", this.Path) + `,`, `Shards:` + fmt.Sprintf("%v", this.Shards) + `,`, `Interval:` + fmt.Sprintf("%v", this.Interval) + `,`, + `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `}`, }, "") return s @@ -4107,6 +4161,7 @@ func (this *LokiInstantRequest) String() string { `Direction:` + fmt.Sprintf("%v", this.Direction) + `,`, `Path:` + fmt.Sprintf("%v", this.Path) + `,`, `Shards:` + fmt.Sprintf("%v", this.Shards) + `,`, + `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `}`, }, "") return s @@ -4689,6 +4744,41 @@ func (m *LokiRequest) Unmarshal(dAtA []byte) error { break } } + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plan", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var v github_com_grafana_loki_pkg_querier_plan.QueryPlan + m.Plan = &v + if err := m.Plan.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) @@ -4909,6 +4999,41 @@ func (m *LokiInstantRequest) Unmarshal(dAtA []byte) error { } m.Shards = append(m.Shards, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plan", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var v github_com_grafana_loki_pkg_querier_plan.QueryPlan + m.Plan = &v + if err := m.Plan.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) diff --git a/pkg/querier/queryrange/queryrange.proto b/pkg/querier/queryrange/queryrange.proto index d5e89eeee47a7..8eb43e34ca160 100644 --- a/pkg/querier/queryrange/queryrange.proto +++ b/pkg/querier/queryrange/queryrange.proto @@ -33,6 +33,7 @@ message LokiRequest { logproto.Direction direction = 6; string path = 7; repeated string shards = 8 [(gogoproto.jsontag) = "shards"]; + bytes plan = 10 [(gogoproto.customtype) = "github.com/grafana/loki/pkg/querier/plan.QueryPlan"]; } message LokiInstantRequest { @@ -45,6 +46,7 @@ message LokiInstantRequest { logproto.Direction direction = 4; string path = 5; repeated string shards = 6 [(gogoproto.jsontag) = "shards"]; + bytes plan = 7 [(gogoproto.customtype) = "github.com/grafana/loki/pkg/querier/plan.QueryPlan"]; } message LokiResponse { diff --git a/pkg/querier/queryrange/querysharding.go b/pkg/querier/queryrange/querysharding.go index 26ec924ce5c4f..1df7bb4616fb9 100644 --- a/pkg/querier/queryrange/querysharding.go +++ b/pkg/querier/queryrange/querysharding.go @@ -185,7 +185,12 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que mapper := logql.NewShardMapper(resolver, ast.metrics) - noop, bytesPerShard, parsed, err := mapper.Parse(r.GetQuery()) + params, err := ParamsFromRequest(r) + if err != nil { + return nil, err + } + + noop, bytesPerShard, parsed, err := mapper.Parse(params.GetExpression()) if err != nil { level.Warn(logger).Log("msg", "failed mapping AST", "err", err.Error(), "query", r.GetQuery()) return nil, err @@ -203,11 +208,6 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que return ast.next.Do(ctx, r) } - params, err := ParamsFromRequest(r) - if err != nil { - return nil, err - } - var path string switch r := r.(type) { case *LokiRequest: @@ -217,7 +217,7 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que default: return nil, fmt.Errorf("expected *LokiRequest or *LokiInstantRequest, got (%T)", r) } - query := ast.ng.Query(ctx, params, parsed) + query := ast.ng.Query(ctx, logql.ParamsWithExpressionOverride{Params: params, ExpressionOverride: parsed}) res, err := query.Exec(ctx) if err != nil { diff --git a/pkg/querier/queryrange/querysharding_test.go b/pkg/querier/queryrange/querysharding_test.go index 8c77afe0410cf..d3d3ce807ac64 100644 --- a/pkg/querier/queryrange/querysharding_test.go +++ b/pkg/querier/queryrange/querysharding_test.go @@ -19,7 +19,9 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" "github.com/grafana/loki/pkg/storage/config" @@ -172,7 +174,12 @@ func Test_astMapper(t *testing.T) { 0, ) - resp, err := mware.Do(user.InjectOrgID(context.Background(), "1"), defaultReq().WithQuery(`{food="bar"}`)) + req := defaultReq() + req.Query = `{foo="bar"}` + req.Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(req.Query), + } + resp, err := mware.Do(user.InjectOrgID(context.Background(), "1"), req) require.Nil(t, err) require.Equal(t, []*definitions.PrometheusResponseHeader{ @@ -311,7 +318,12 @@ func Test_astMapper_QuerySizeLimits(t *testing.T) { 0, ) - _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), defaultReq().WithQuery(tc.query)) + req := defaultReq() + req.Query = tc.query + req.Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(tc.query), + } + _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), req) if err != nil { require.ErrorContains(t, err, tc.err) } @@ -344,7 +356,13 @@ func Test_ShardingByPass(t *testing.T) { 0, ) - _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), defaultReq().WithQuery(`1+1`)) + req := defaultReq() + req.Query = `1+1` + req.Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(req.Query), + } + + _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), req) require.Nil(t, err) require.Equal(t, called, 1) } @@ -437,6 +455,9 @@ func Test_InstantSharding(t *testing.T) { Query: `rate({app="foo"}[1m])`, TimeTs: util.TimeFromMillis(10), Path: "/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`rate({app="foo"}[1m])`), + }, }) require.NoError(t, err) require.Equal(t, 3, called, "expected 3 calls but got {}", called) @@ -703,6 +724,13 @@ func TestShardingAcrossConfigs_ASTMapper(t *testing.T) { 0, ) + // currently all the tests call `defaultReq()` which creates an instance of the type LokiRequest + // if in the future that isn't true, we need another way to access the Plan field of an arbitrary query type + // or we should set the Plan in calls to `GetExpression` if the Plan is nil by calling `ParseExpr` or similar + tc.req.(*LokiRequest).Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(tc.req.GetQuery()), + } + resp, err := mware.Do(user.InjectOrgID(context.Background(), "1"), tc.req) require.Nil(t, err) @@ -830,12 +858,16 @@ func Test_ASTMapper_MaxLookBackPeriod(t *testing.T) { 0, ) + q := `{cluster="dev-us-central-0"}` lokiReq := &LokiInstantRequest{ - Query: `{cluster="dev-us-central-0"}`, + Query: q, Limit: 1000, TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(q), + }, } ctx := user.InjectOrgID(context.Background(), "foo") diff --git a/pkg/querier/queryrange/roundtrip_test.go b/pkg/querier/queryrange/roundtrip_test.go index 4c19a1ffc1202..b91e088041db9 100644 --- a/pkg/querier/queryrange/roundtrip_test.go +++ b/pkg/querier/queryrange/roundtrip_test.go @@ -23,8 +23,10 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" base "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/config" @@ -332,12 +334,16 @@ func TestInstantQueryTripperware(t *testing.T) { } require.NoError(t, err) + q := `sum by (job) (bytes_rate({cluster="dev-us-central-0"}[15m]))` lreq := &LokiInstantRequest{ - Query: `sum by (job) (bytes_rate({cluster="dev-us-central-0"}[15m]))`, + Query: q, Limit: 1000, TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(q), + }, } ctx := user.InjectOrgID(context.Background(), "1") @@ -1101,6 +1107,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[2h]))`), + }, }, expectedSplitStats: 2, // [2h] interval split by 1h configured split interval expectedShardStats: 8, // 2 time splits * 4 row shards @@ -1113,6 +1122,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[1h]))`), + }, }, expectedSplitStats: 0, // [1h] interval not split expectedShardStats: 4, // 4 row shards @@ -1127,6 +1139,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { EndTs: testTime, Direction: logproto.FORWARD, Path: "/query_range", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[1h]))`), + }, }, expectedSplitStats: 3, // 2 hour range interval split based on the base hour + the remainder expectedShardStats: 12, // 3 time splits * 4 row shards @@ -1141,6 +1156,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { EndTs: testTime, Direction: logproto.FORWARD, Path: "/query_range", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[1h]))`), + }, }, expectedSplitStats: 0, // 1 minute range interval not split expectedShardStats: 4, // 4 row shards diff --git a/pkg/querier/queryrange/split_by_interval.go b/pkg/querier/queryrange/split_by_interval.go index da8326a678ec5..b4245375bce66 100644 --- a/pkg/querier/queryrange/split_by_interval.go +++ b/pkg/querier/queryrange/split_by_interval.go @@ -383,6 +383,7 @@ func splitMetricByTime(r queryrangebase.Request, interval time.Duration) ([]quer Path: lokiReq.Path, StartTs: start, EndTs: end, + Plan: lokiReq.Plan, }) }) @@ -403,6 +404,7 @@ func splitMetricByTime(r queryrangebase.Request, interval time.Duration) ([]quer Path: lokiReq.Path, StartTs: start, EndTs: end, + Plan: lokiReq.Plan, }) } diff --git a/pkg/querier/queryrange/split_by_range.go b/pkg/querier/queryrange/split_by_range.go index e3640761d57ec..6845846d4deaa 100644 --- a/pkg/querier/queryrange/split_by_range.go +++ b/pkg/querier/queryrange/split_by_range.go @@ -47,6 +47,11 @@ func NewSplitByRangeMiddleware(logger log.Logger, engineOpts logql.EngineOpts, l func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) (queryrangebase.Response, error) { logger := util_log.WithContext(ctx, s.logger) + params, err := ParamsFromRequest(request) + if err != nil { + return nil, err + } + tenants, err := tenant.TenantIDs(ctx) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) @@ -64,7 +69,7 @@ func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) ( return nil, err } - noop, parsed, err := mapper.Parse(request.GetQuery()) + noop, parsed, err := mapper.Parse(params.GetExpression()) if err != nil { level.Warn(logger).Log("msg", "failed mapping AST", "err", err.Error(), "query", request.GetQuery()) return nil, err @@ -80,16 +85,11 @@ func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) ( queryStatsCtx := stats.FromContext(ctx) queryStatsCtx.AddSplitQueries(int64(mapperStats.GetSplitQueries())) - params, err := ParamsFromRequest(request) - if err != nil { - return nil, err - } - if _, ok := request.(*LokiInstantRequest); !ok { - return nil, fmt.Errorf("expected *LokiInstantRequest") + return nil, fmt.Errorf("expected *LokiInstantRequest, got %T", request) } - query := s.ng.Query(ctx, params, parsed) + query := s.ng.Query(ctx, logql.ParamsWithExpressionOverride{Params: params, ExpressionOverride: parsed}) res, err := query.Exec(ctx) if err != nil { diff --git a/pkg/querier/queryrange/split_by_range_test.go b/pkg/querier/queryrange/split_by_range_test.go index c3b4587a1dbb1..ef25e3f910fb3 100644 --- a/pkg/querier/queryrange/split_by_range_test.go +++ b/pkg/querier/queryrange/split_by_range_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" - "github.com/grafana/loki/pkg/loghttp" - "github.com/go-kit/log" "github.com/grafana/dskit/user" "github.com/stretchr/testify/require" + "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" ) @@ -37,6 +38,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum(bytes_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum(bytes_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum(bytes_over_time({app="foo"}[1m]))`, 1), @@ -50,6 +54,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum by (bar) (bytes_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (bar) (bytes_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum by (bar)(bytes_over_time({app="foo"}[1m]))`, 10), @@ -63,6 +70,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum(count_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum(count_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum(count_over_time({app="foo"}[1m]))`, 1), @@ -76,6 +86,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum by (bar) (count_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (bar) (count_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum by (bar)(count_over_time({app="foo"}[1m]))`, 0), @@ -89,6 +102,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum(sum_over_time({app="foo"} | unwrap bar [3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum(sum_over_time({app="foo"} | unwrap bar [3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum(sum_over_time({app="foo"} | unwrap bar[1m]))`, 1), @@ -102,6 +118,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum by (bar) (sum_over_time({app="foo"} | unwrap bar [3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (bar) (sum_over_time({app="foo"} | unwrap bar [3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum by (bar)(sum_over_time({app="foo"} | unwrap bar[1m]))`, 1), @@ -140,6 +159,9 @@ func subQueryRequestResponse(expectedSubQuery string, sampleValue float64) query Query: expectedSubQuery, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(expectedSubQuery), + }, }, Response: &LokiPromResponse{ Response: &queryrangebase.PrometheusResponse{ diff --git a/pkg/querier/queryrange/stats.go b/pkg/querier/queryrange/stats.go index 0233d886c98f2..71f93959c3b69 100644 --- a/pkg/querier/queryrange/stats.go +++ b/pkg/querier/queryrange/stats.go @@ -53,13 +53,13 @@ func recordQueryMetrics(data *queryData) { case queryTypeLog, queryTypeMetric: logql.RecordRangeAndInstantQueryMetrics(data.ctx, logger, data.params, data.status, *data.statistics, data.result) case queryTypeLabel: - logql.RecordLabelQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.label, data.params.Query(), data.status, *data.statistics) + logql.RecordLabelQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.label, data.params.QueryString(), data.status, *data.statistics) case queryTypeSeries: logql.RecordSeriesQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.match, data.status, []string{}, *data.statistics) case queryTypeStats: - logql.RecordStatsQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.Query(), data.status, *data.statistics) + logql.RecordStatsQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.QueryString(), data.status, *data.statistics) case queryTypeVolume: - logql.RecordVolumeQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.Query(), data.params.Limit(), data.params.Step(), data.status, *data.statistics) + logql.RecordVolumeQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.QueryString(), data.params.Limit(), data.params.Step(), data.status, *data.statistics) default: level.Error(logger).Log("msg", "failed to record query metrics", "err", fmt.Errorf("expected one of the *LokiRequest, *LokiInstantRequest, *LokiSeriesRequest, *LokiLabelNamesRequest, got %s", data.queryType)) } diff --git a/pkg/querier/queryrange/stats_test.go b/pkg/querier/queryrange/stats_test.go index 54c9004d88cec..28f8d12de7f6d 100644 --- a/pkg/querier/queryrange/stats_test.go +++ b/pkg/querier/queryrange/stats_test.go @@ -30,7 +30,7 @@ func TestStatsCollectorMiddleware(t *testing.T) { Query: "foo", StartTs: now, }) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, true, data.recorded) require.Equal(t, now, data.params.Start()) require.Nil(t, data.statistics) @@ -60,7 +60,7 @@ func TestStatsCollectorMiddleware(t *testing.T) { Query: "foo", StartTs: now, }) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, true, data.recorded) require.Equal(t, now, data.params.Start()) require.Equal(t, int32(10), data.statistics.Ingester.TotalReached) @@ -108,7 +108,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusOK), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, logproto.BACKWARD, data.params.Direction()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, stats.Result{}, *data.statistics) @@ -129,7 +129,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusTeapot), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, logproto.BACKWARD, data.params.Direction()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, statsResult, *data.statistics) @@ -151,7 +151,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusTeapot), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, logproto.BACKWARD, data.params.Direction()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, statsResult, *data.statistics) @@ -173,7 +173,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusTeapot), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, statsResult, *data.statistics) require.Equal(t, streams, data.result) diff --git a/pkg/querier/worker/util_test.go b/pkg/querier/worker/util_test.go index a0213e3bb708b..25dd8127a0da4 100644 --- a/pkg/querier/worker/util_test.go +++ b/pkg/querier/worker/util_test.go @@ -61,7 +61,7 @@ func TestHandleQueryRequest(t *testing.T) { } { t.Run(name, func(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "1") - request, err := queryrange.DefaultCodec.QueryRequestWrap(ctx, &queryrange.LokiRequest{}) + request, err := queryrange.DefaultCodec.QueryRequestWrap(ctx, &queryrange.LokiRequest{Query: `{app="foo"}`}) require.NoError(t, err) mockHandler := HandlerFunc(func(context.Context, queryrangebase.Request) (queryrangebase.Response, error) { diff --git a/pkg/ruler/evaluator_local.go b/pkg/ruler/evaluator_local.go index fed0f2f02ef11..91efd5a14d995 100644 --- a/pkg/ruler/evaluator_local.go +++ b/pkg/ruler/evaluator_local.go @@ -28,7 +28,7 @@ func NewLocalEvaluator(engine *logql.Engine, logger log.Logger) (*LocalEvaluator } func (l *LocalEvaluator) Eval(ctx context.Context, qs string, now time.Time) (*logqlmodel.Result, error) { - params := logql.NewLiteralParams( + params, err := logql.NewLiteralParams( qs, now, now, @@ -38,6 +38,9 @@ func (l *LocalEvaluator) Eval(ctx context.Context, qs string, now time.Time) (*l 0, nil, ) + if err != nil { + return nil, err + } q := l.engine.Query(params) res, err := q.Exec(ctx)