From 64eb68e39b4eac8d2c1488d26d3643f54c2939f2 Mon Sep 17 00:00:00 2001 From: milesbryant Date: Fri, 15 Nov 2024 09:49:40 +0000 Subject: [PATCH] Enable Thanos Query Stats Propagation & cache response headers Thanos Query gives us some really nice detailed internal stats - but thanos-query-frontend annoyingly blats them by trying to parse the response as a Prometheus response, which has different fields and a different structure. In addition, thanos-query-frontend doesn't even pass the &stats=all parameter through if you try to request it. This change fixes this by propagating the stats request parameter to the downstream, and decoding/encoding the Thanos Query stats structure properly. As an extra, we also set a response header if we get a cache hit so that upstreams can use this. Signed-off-by: milesbryant --- cmd/thanos/query_frontend.go | 4 +- .../frontend/downstream_roundtripper.go | 16 +- internal/cortex/frontend/transport/handler.go | 90 +-- internal/cortex/querier/querier.go | 1 - .../cortex/querier/queryrange/query_range.go | 76 +- .../querier/queryrange/queryrange.pb.go | 675 ++++++++++++++---- .../querier/queryrange/queryrange.proto | 20 +- .../querier/queryrange/results_cache.go | 63 +- pkg/queryfrontend/queryrange_codec.go | 2 + pkg/queryfrontend/response.go | 4 + pkg/queryfrontend/roundtrip.go | 2 + 11 files changed, 668 insertions(+), 285 deletions(-) diff --git a/cmd/thanos/query_frontend.go b/cmd/thanos/query_frontend.go index 5b6fc0b902..5fa7cf3c5e 100644 --- a/cmd/thanos/query_frontend.go +++ b/cmd/thanos/query_frontend.go @@ -145,8 +145,6 @@ func registerQueryFrontend(app *extkingpin.App) { cmd.Flag("query-frontend.log-queries-longer-than", "Log queries that are slower than the specified duration. "+ "Set to 0 to disable. Set to < 0 to enable on all queries.").Default("0").DurationVar(&cfg.CortexHandlerConfig.LogQueriesLongerThan) - cmd.Flag("query-frontend.query-stats-enabled", "True to enable query statistics tracking. "+ - "When enabled, a message with some statistics is logged for every query.").Default("false").BoolVar(&cfg.CortexHandlerConfig.QueryStatsEnabled) cmd.Flag("query-frontend.org-id-header", "Deprecation Warning - This flag will be soon deprecated in favor of query-frontend.tenant-header"+ " and both flags cannot be used at the same time. "+ @@ -313,7 +311,7 @@ func runQueryFrontend( return err } - roundTripper, err := cortexfrontend.NewDownstreamRoundTripper(cfg.DownstreamURL, downstreamTripper, cfg.CortexHandlerConfig.QueryStatsEnabled) + roundTripper, err := cortexfrontend.NewDownstreamRoundTripper(cfg.DownstreamURL, downstreamTripper) if err != nil { return errors.Wrap(err, "setup downstream roundtripper") } diff --git a/internal/cortex/frontend/downstream_roundtripper.go b/internal/cortex/frontend/downstream_roundtripper.go index 9c165f4476..9629d83ecf 100644 --- a/internal/cortex/frontend/downstream_roundtripper.go +++ b/internal/cortex/frontend/downstream_roundtripper.go @@ -13,18 +13,17 @@ import ( // RoundTripper that forwards requests to downstream URL. type downstreamRoundTripper struct { - downstreamURL *url.URL - transport http.RoundTripper - queryStatsEnabled bool + downstreamURL *url.URL + transport http.RoundTripper } -func NewDownstreamRoundTripper(downstreamURL string, transport http.RoundTripper, queryStatsEnabled bool) (http.RoundTripper, error) { +func NewDownstreamRoundTripper(downstreamURL string, transport http.RoundTripper) (http.RoundTripper, error) { u, err := url.Parse(downstreamURL) if err != nil { return nil, err } - return &downstreamRoundTripper{downstreamURL: u, transport: transport, queryStatsEnabled: queryStatsEnabled}, nil + return &downstreamRoundTripper{downstreamURL: u, transport: transport}, nil } func (d downstreamRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { @@ -37,13 +36,6 @@ func (d downstreamRoundTripper) RoundTrip(r *http.Request) (*http.Response, erro } } - if d.queryStatsEnabled { - // add &stats query param to get thanos-query to add query statistics to log - q := r.URL.Query() - q.Set("stats", "true") - r.URL.RawQuery = q.Encode() - } - r.URL.Scheme = d.downstreamURL.Scheme r.URL.Host = d.downstreamURL.Host r.URL.Path = path.Join(d.downstreamURL.Path, r.URL.Path) diff --git a/internal/cortex/frontend/transport/handler.go b/internal/cortex/frontend/transport/handler.go index 27e7932662..4d62ec059d 100644 --- a/internal/cortex/frontend/transport/handler.go +++ b/internal/cortex/frontend/transport/handler.go @@ -6,10 +6,8 @@ package transport import ( "bytes" "context" - "encoding/json" "errors" "fmt" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/prometheus/util/stats" "io" "net/http" @@ -44,7 +42,6 @@ var ( type HandlerConfig struct { LogQueriesLongerThan time.Duration `yaml:"log_queries_longer_than"` MaxBodySize int64 `yaml:"max_body_size"` - QueryStatsEnabled bool `yaml:"query_stats_enabled"` } // Handler accepts queries and forwards them to RoundTripper. It can log slow queries, @@ -68,27 +65,6 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge roundTripper: roundTripper, } - if cfg.QueryStatsEnabled { - h.querySeconds = promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "thanos_query_frontend_query_seconds", - Help: "Total amount of wall clock time spend processing queries.", - Buckets: []float64{0.01, 0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 360}, - }, []string{"user"}) - - h.querySamplesTotal = promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "thanos_query_frontend_query_total_fetched_samples", - Help: "Number of samples touched to execute a query.", - Buckets: []float64{1, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000}, - }, []string{"user"}) - - h.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(func(user string) { - h.querySeconds.DeleteLabelValues(user) - h.querySamplesTotal.DeleteLabelValues(user) - }) - // If cleaner stops or fail, we will simply not clean the metrics for inactive users. - _ = h.activeUsers.StartAsync(context.Background()) - } - return h } @@ -129,38 +105,15 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(resp.StatusCode) - var respBuf bytes.Buffer - if f.cfg.QueryStatsEnabled { - // Buffer the response body for query stat tracking later - resp.Body = io.NopCloser(io.TeeReader(resp.Body, &respBuf)) - } - // log copy response body error so that we will know even though success response code returned bytesCopied, err := io.Copy(w, resp.Body) if err != nil && !errors.Is(err, syscall.EPIPE) { level.Error(util_log.WithContext(r.Context(), f.log)).Log("msg", "write response body error", "bytesCopied", bytesCopied, "err", err) } - if f.cfg.QueryStatsEnabled { - // Parse the stats field out of the response body - var statsResponse ResponseWithStats - if err := json.Unmarshal(respBuf.Bytes(), &statsResponse); err == nil { - if statsResponse.Data.Stats != nil { - queryString = f.parseRequestQueryString(r, buf) - f.reportQueryStats(r, queryString, queryResponseTime, statsResponse.Data.Stats) - } else { - // Don't fail the request if the stats are nil, just log a warning - level.Warn(util_log.WithContext(r.Context(), f.log)).Log("msg", "error parsing query stats", "err", errors.New("stats are nil")) - } - } else { - // Don't fail the request if the stats are nil, just log a warning - level.Warn(util_log.WithContext(r.Context(), f.log)).Log("msg", "error parsing query stats", "err", err) - } - } - // Check whether we should parse the query string. shouldReportSlowQuery := f.cfg.LogQueriesLongerThan != 0 && queryResponseTime > f.cfg.LogQueriesLongerThan - if shouldReportSlowQuery || f.cfg.QueryStatsEnabled { + if shouldReportSlowQuery { queryString = f.parseRequestQueryString(r, buf) } @@ -203,47 +156,6 @@ func (f *Handler) reportSlowQuery(r *http.Request, responseHeaders http.Header, level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } -func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, queryResponseTime time.Duration, stats *stats.BuiltinStats) { - remoteUser, _, _ := r.BasicAuth() - - // Log stats. - logMessage := []interface{}{ - "msg", "query stats", - "component", "query-frontend", - "method", r.Method, - "path", r.URL.Path, - "remote_user", remoteUser, - "remote_addr", r.RemoteAddr, - "response_time", queryResponseTime, - "query_timings_preparation_time", stats.Timings.QueryPreparationTime, - "query_timings_eval_total_time", stats.Timings.EvalTotalTime, - "query_timings_exec_total_time", stats.Timings.ExecTotalTime, - "query_timings_exec_queue_time", stats.Timings.ExecQueueTime, - "query_timings_inner_eval_time", stats.Timings.InnerEvalTime, - "query_timings_result_sort_time", stats.Timings.ResultSortTime, - } - if stats.Samples != nil { - samples := stats.Samples - - logMessage = append(logMessage, []interface{}{ - "total_queryable_samples", samples.TotalQueryableSamples, - "peak_samples", samples.PeakSamples, - }...) - } - - logMessage = append(logMessage, formatQueryString(queryString)...) - - level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) - - // Record metrics. - if f.querySeconds != nil { - f.querySeconds.WithLabelValues(remoteUser).Observe(queryResponseTime.Seconds()) - } - if f.querySamplesTotal != nil && stats.Samples != nil { - f.querySamplesTotal.WithLabelValues(remoteUser).Observe(float64(stats.Samples.TotalQueryableSamples)) - } -} - func (f *Handler) parseRequestQueryString(r *http.Request, bodyBuf bytes.Buffer) url.Values { // Use previously buffered body. r.Body = io.NopCloser(&bodyBuf) diff --git a/internal/cortex/querier/querier.go b/internal/cortex/querier/querier.go index 9d3fad079e..d586f617e1 100644 --- a/internal/cortex/querier/querier.go +++ b/internal/cortex/querier/querier.go @@ -21,7 +21,6 @@ type Config struct { QueryIngestersWithin time.Duration `yaml:"query_ingesters_within"` QueryStoreForLabels bool `yaml:"query_store_for_labels_enabled"` AtModifierEnabled bool `yaml:"at_modifier_enabled"` - EnablePerStepStats bool `yaml:"per_step_stats_enabled"` // QueryStoreAfter the time after which queries should also be sent to the store and not just ingesters. QueryStoreAfter time.Duration `yaml:"query_store_after"` diff --git a/internal/cortex/querier/queryrange/query_range.go b/internal/cortex/querier/queryrange/query_range.go index 5304f9c63d..6124487c55 100644 --- a/internal/cortex/querier/queryrange/query_range.go +++ b/internal/cortex/querier/queryrange/query_range.go @@ -107,6 +107,8 @@ type Response interface { GetHeaders() []*PrometheusResponseHeader // GetStats returns the Prometheus query stats in the response. GetStats() *PrometheusResponseStats + // AddHeader adds a HTTP header to the response + AddHeader(key, value string) } type prometheusCodec struct{} @@ -184,6 +186,14 @@ func (resp *PrometheusInstantQueryResponse) GetStats() *PrometheusResponseStats return resp.Data.Stats } +func (resp *PrometheusResponse) AddHeader(key, value string) { + resp.Headers = append(resp.Headers, &PrometheusResponseHeader{Name: key, Values: []string{value}}) +} + +func (resp *PrometheusInstantQueryResponse) AddHeader(key, value string) { + resp.Headers = append(resp.Headers, &PrometheusResponseHeader{Name: key, Values: []string{value}}) +} + // NewEmptyPrometheusResponse returns an empty successful Prometheus query range response. func NewEmptyPrometheusResponse() *PrometheusResponse { return &PrometheusResponse{ @@ -217,9 +227,25 @@ func (prometheusCodec) MergeResponse(_ Request, responses ...Response) (Response // we need to pass on all the headers for results cache gen numbers. var resultsCacheGenNumberHeaderValues []string + var headers []*PrometheusResponseHeader + // merge headers for _, res := range responses { promResponses = append(promResponses, res.(*PrometheusResponse)) resultsCacheGenNumberHeaderValues = append(resultsCacheGenNumberHeaderValues, getHeaderValuesWithName(res, ResultsCacheGenNumberHeaderName)...) + for _, newHeader := range res.GetHeaders() { + found := false + for _, existing := range headers { + if existing.Name == newHeader.Name { + // if headers match, overwrite with the new header + existing = newHeader + found = true + } + } + if !found { + // new header doesn't exist in existing list, so add it + headers = append(headers, newHeader) + } + } } // Merge the responses. @@ -244,10 +270,14 @@ func (prometheusCodec) MergeResponse(_ Request, responses ...Response) (Response } if len(resultsCacheGenNumberHeaderValues) != 0 { - response.Headers = []*PrometheusResponseHeader{{ + response.Headers = append(response.Headers, &PrometheusResponseHeader{ Name: ResultsCacheGenNumberHeaderName, Values: resultsCacheGenNumberHeaderValues, - }} + }) + } + + for _, h := range headers { + response.Headers = append(response.Headers, h) } return &response, nil @@ -419,6 +449,16 @@ func (prometheusCodec) EncodeResponse(ctx context.Context, res Response) (*http. StatusCode: http.StatusOK, ContentLength: int64(len(b)), } + + // Copy Prometheus headers into http response + for _, h := range a.Headers { + for _, v := range h.Values { + if strings.HasPrefix(h.Name, "X-") { + // Only copy X- headers, otherwise we might corrupt HTTP protocol headers (like Content-Length) + resp.Header.Add(h.Name, v) + } + } + } return &resp, nil } @@ -665,6 +705,23 @@ func (s *PrometheusInstantQueryData) MarshalJSON() ([]byte, error) { func StatsMerge(resps []Response) *PrometheusResponseStats { output := map[int64]*PrometheusResponseQueryableSamplesStatsPerStep{} hasStats := false + + result := &PrometheusResponseStats{ + Timings: &PrometheusResponseStats_Timings{ + EvalTotalTime: 0, + ResultSortTime: 0, + QueryPreparationTime: 0, + InnerEvalTime: 0, + ExecQueueTime: 0, + ExecTotalTime: 0, + }, + Samples: &PrometheusResponseStats_Samples{ + TotalQueryableSamples: 0, + PeakSamples: 0, + TotalQueryableSamplesPerStep: []*PrometheusResponseQueryableSamplesStatsPerStep{}, + }, + } + for _, resp := range resps { stats := resp.GetStats() if stats == nil { @@ -679,6 +736,19 @@ func StatsMerge(resps []Response) *PrometheusResponseStats { for _, s := range stats.Samples.TotalQueryableSamplesPerStep { output[s.GetTimestampMs()] = s } + + // add all the stats + result.Timings.EvalTotalTime += stats.Timings.EvalTotalTime + result.Timings.ResultSortTime += stats.Timings.ResultSortTime + result.Timings.QueryPreparationTime += stats.Timings.QueryPreparationTime + result.Timings.InnerEvalTime += stats.Timings.InnerEvalTime + result.Timings.ExecQueueTime += stats.Timings.ExecQueueTime + result.Timings.ExecTotalTime += stats.Timings.ExecTotalTime + + result.Samples.TotalQueryableSamples += stats.Samples.TotalQueryableSamples + if stats.Samples.PeakSamples > result.Samples.PeakSamples { + result.Samples.PeakSamples = stats.Samples.PeakSamples + } } if !hasStats { @@ -692,10 +762,8 @@ func StatsMerge(resps []Response) *PrometheusResponseStats { sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - result := &PrometheusResponseStats{Samples: &PrometheusResponseSamplesStats{}} for _, key := range keys { result.Samples.TotalQueryableSamplesPerStep = append(result.Samples.TotalQueryableSamplesPerStep, output[key]) - result.Samples.TotalQueryableSamples += output[key].Value } return result diff --git a/internal/cortex/querier/queryrange/queryrange.pb.go b/internal/cortex/querier/queryrange/queryrange.pb.go index 0b97731181..e09d58723f 100644 --- a/internal/cortex/querier/queryrange/queryrange.pb.go +++ b/internal/cortex/querier/queryrange/queryrange.pb.go @@ -763,10 +763,11 @@ func (m *Matrix) GetSampleStreams() []*SampleStream { } type PrometheusResponseStats struct { - Samples *PrometheusResponseSamplesStats `protobuf:"bytes,1,opt,name=samples,proto3" json:"samples"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Timings *PrometheusResponseStats_Timings `protobuf:"bytes,1,opt,name=timings,proto3" json:"timings"` + Samples *PrometheusResponseStats_Samples `protobuf:"bytes,2,opt,name=samples,proto3" json:"samples"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *PrometheusResponseStats) Reset() { *m = PrometheusResponseStats{} } @@ -802,33 +803,128 @@ func (m *PrometheusResponseStats) XXX_DiscardUnknown() { var xxx_messageInfo_PrometheusResponseStats proto.InternalMessageInfo -func (m *PrometheusResponseStats) GetSamples() *PrometheusResponseSamplesStats { +func (m *PrometheusResponseStats) GetTimings() *PrometheusResponseStats_Timings { + if m != nil { + return m.Timings + } + return nil +} + +func (m *PrometheusResponseStats) GetSamples() *PrometheusResponseStats_Samples { if m != nil { return m.Samples } return nil } -type PrometheusResponseSamplesStats struct { - TotalQueryableSamples int64 `protobuf:"varint,1,opt,name=totalQueryableSamples,proto3" json:"totalQueryableSamples"` - TotalQueryableSamplesPerStep []*PrometheusResponseQueryableSamplesStatsPerStep `protobuf:"bytes,2,rep,name=totalQueryableSamplesPerStep,proto3" json:"totalQueryableSamplesPerStep"` +type PrometheusResponseStats_Timings struct { + EvalTotalTime float32 `protobuf:"fixed32,1,opt,name=EvalTotalTime,proto3" json:"evalTotalTime"` + ResultSortTime float32 `protobuf:"fixed32,2,opt,name=ResultSortTime,proto3" json:"resultSortTime"` + QueryPreparationTime float32 `protobuf:"fixed32,3,opt,name=QueryPreparationTime,proto3" json:"queryPreparationTime"` + InnerEvalTime float32 `protobuf:"fixed32,4,opt,name=InnerEvalTime,proto3" json:"innerEvalTime"` + ExecQueueTime float32 `protobuf:"fixed32,5,opt,name=ExecQueueTime,proto3" json:"execQueueTime"` + ExecTotalTime float32 `protobuf:"fixed32,6,opt,name=ExecTotalTime,proto3" json:"execTotalTime"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PrometheusResponseStats_Timings) Reset() { *m = PrometheusResponseStats_Timings{} } +func (m *PrometheusResponseStats_Timings) String() string { return proto.CompactTextString(m) } +func (*PrometheusResponseStats_Timings) ProtoMessage() {} +func (*PrometheusResponseStats_Timings) Descriptor() ([]byte, []int) { + return fileDescriptor_9af7607b46ac39b7, []int{10, 0} +} +func (m *PrometheusResponseStats_Timings) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PrometheusResponseStats_Timings) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PrometheusResponseStats_Timings.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PrometheusResponseStats_Timings) XXX_Merge(src proto.Message) { + xxx_messageInfo_PrometheusResponseStats_Timings.Merge(m, src) +} +func (m *PrometheusResponseStats_Timings) XXX_Size() int { + return m.Size() +} +func (m *PrometheusResponseStats_Timings) XXX_DiscardUnknown() { + xxx_messageInfo_PrometheusResponseStats_Timings.DiscardUnknown(m) +} + +var xxx_messageInfo_PrometheusResponseStats_Timings proto.InternalMessageInfo + +func (m *PrometheusResponseStats_Timings) GetEvalTotalTime() float32 { + if m != nil { + return m.EvalTotalTime + } + return 0 +} + +func (m *PrometheusResponseStats_Timings) GetResultSortTime() float32 { + if m != nil { + return m.ResultSortTime + } + return 0 +} + +func (m *PrometheusResponseStats_Timings) GetQueryPreparationTime() float32 { + if m != nil { + return m.QueryPreparationTime + } + return 0 +} + +func (m *PrometheusResponseStats_Timings) GetInnerEvalTime() float32 { + if m != nil { + return m.InnerEvalTime + } + return 0 +} + +func (m *PrometheusResponseStats_Timings) GetExecQueueTime() float32 { + if m != nil { + return m.ExecQueueTime + } + return 0 +} + +func (m *PrometheusResponseStats_Timings) GetExecTotalTime() float32 { + if m != nil { + return m.ExecTotalTime + } + return 0 +} + +type PrometheusResponseStats_Samples struct { + TotalQueryableSamples int64 `protobuf:"varint,1,opt,name=TotalQueryableSamples,proto3" json:"totalQueryableSamples"` + PeakSamples int64 `protobuf:"varint,2,opt,name=PeakSamples,proto3" json:"peakSamples"` + TotalQueryableSamplesPerStep []*PrometheusResponseQueryableSamplesStatsPerStep `protobuf:"bytes,3,rep,name=totalQueryableSamplesPerStep,proto3" json:"totalQueryableSamplesPerStep"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *PrometheusResponseSamplesStats) Reset() { *m = PrometheusResponseSamplesStats{} } -func (m *PrometheusResponseSamplesStats) String() string { return proto.CompactTextString(m) } -func (*PrometheusResponseSamplesStats) ProtoMessage() {} -func (*PrometheusResponseSamplesStats) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{11} +func (m *PrometheusResponseStats_Samples) Reset() { *m = PrometheusResponseStats_Samples{} } +func (m *PrometheusResponseStats_Samples) String() string { return proto.CompactTextString(m) } +func (*PrometheusResponseStats_Samples) ProtoMessage() {} +func (*PrometheusResponseStats_Samples) Descriptor() ([]byte, []int) { + return fileDescriptor_9af7607b46ac39b7, []int{10, 1} } -func (m *PrometheusResponseSamplesStats) XXX_Unmarshal(b []byte) error { +func (m *PrometheusResponseStats_Samples) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *PrometheusResponseSamplesStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *PrometheusResponseStats_Samples) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_PrometheusResponseSamplesStats.Marshal(b, m, deterministic) + return xxx_messageInfo_PrometheusResponseStats_Samples.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -838,26 +934,33 @@ func (m *PrometheusResponseSamplesStats) XXX_Marshal(b []byte, deterministic boo return b[:n], nil } } -func (m *PrometheusResponseSamplesStats) XXX_Merge(src proto.Message) { - xxx_messageInfo_PrometheusResponseSamplesStats.Merge(m, src) +func (m *PrometheusResponseStats_Samples) XXX_Merge(src proto.Message) { + xxx_messageInfo_PrometheusResponseStats_Samples.Merge(m, src) } -func (m *PrometheusResponseSamplesStats) XXX_Size() int { +func (m *PrometheusResponseStats_Samples) XXX_Size() int { return m.Size() } -func (m *PrometheusResponseSamplesStats) XXX_DiscardUnknown() { - xxx_messageInfo_PrometheusResponseSamplesStats.DiscardUnknown(m) +func (m *PrometheusResponseStats_Samples) XXX_DiscardUnknown() { + xxx_messageInfo_PrometheusResponseStats_Samples.DiscardUnknown(m) } -var xxx_messageInfo_PrometheusResponseSamplesStats proto.InternalMessageInfo +var xxx_messageInfo_PrometheusResponseStats_Samples proto.InternalMessageInfo -func (m *PrometheusResponseSamplesStats) GetTotalQueryableSamples() int64 { +func (m *PrometheusResponseStats_Samples) GetTotalQueryableSamples() int64 { if m != nil { return m.TotalQueryableSamples } return 0 } -func (m *PrometheusResponseSamplesStats) GetTotalQueryableSamplesPerStep() []*PrometheusResponseQueryableSamplesStatsPerStep { +func (m *PrometheusResponseStats_Samples) GetPeakSamples() int64 { + if m != nil { + return m.PeakSamples + } + return 0 +} + +func (m *PrometheusResponseStats_Samples) GetTotalQueryableSamplesPerStep() []*PrometheusResponseQueryableSamplesStatsPerStep { if m != nil { return m.TotalQueryableSamplesPerStep } @@ -880,7 +983,7 @@ func (m *PrometheusResponseQueryableSamplesStatsPerStep) String() string { } func (*PrometheusResponseQueryableSamplesStatsPerStep) ProtoMessage() {} func (*PrometheusResponseQueryableSamplesStatsPerStep) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{12} + return fileDescriptor_9af7607b46ac39b7, []int{11} } func (m *PrometheusResponseQueryableSamplesStatsPerStep) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -936,7 +1039,7 @@ func (m *SampleStream) Reset() { *m = SampleStream{} } func (m *SampleStream) String() string { return proto.CompactTextString(m) } func (*SampleStream) ProtoMessage() {} func (*SampleStream) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{13} + return fileDescriptor_9af7607b46ac39b7, []int{12} } func (m *SampleStream) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -993,7 +1096,7 @@ func (m *Sample) Reset() { *m = Sample{} } func (m *Sample) String() string { return proto.CompactTextString(m) } func (*Sample) ProtoMessage() {} func (*Sample) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{14} + return fileDescriptor_9af7607b46ac39b7, []int{13} } func (m *Sample) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1055,7 +1158,7 @@ func (m *StringSample) Reset() { *m = StringSample{} } func (m *StringSample) String() string { return proto.CompactTextString(m) } func (*StringSample) ProtoMessage() {} func (*StringSample) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{15} + return fileDescriptor_9af7607b46ac39b7, []int{14} } func (m *StringSample) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1110,7 +1213,7 @@ func (m *SampleHistogramPair) Reset() { *m = SampleHistogramPair{} } func (m *SampleHistogramPair) String() string { return proto.CompactTextString(m) } func (*SampleHistogramPair) ProtoMessage() {} func (*SampleHistogramPair) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{16} + return fileDescriptor_9af7607b46ac39b7, []int{15} } func (m *SampleHistogramPair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1166,7 +1269,7 @@ func (m *SampleHistogram) Reset() { *m = SampleHistogram{} } func (m *SampleHistogram) String() string { return proto.CompactTextString(m) } func (*SampleHistogram) ProtoMessage() {} func (*SampleHistogram) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{17} + return fileDescriptor_9af7607b46ac39b7, []int{16} } func (m *SampleHistogram) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1230,7 +1333,7 @@ func (m *HistogramBucket) Reset() { *m = HistogramBucket{} } func (m *HistogramBucket) String() string { return proto.CompactTextString(m) } func (*HistogramBucket) ProtoMessage() {} func (*HistogramBucket) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{18} + return fileDescriptor_9af7607b46ac39b7, []int{17} } func (m *HistogramBucket) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1300,7 +1403,7 @@ func (m *CachedResponse) Reset() { *m = CachedResponse{} } func (m *CachedResponse) String() string { return proto.CompactTextString(m) } func (*CachedResponse) ProtoMessage() {} func (*CachedResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{19} + return fileDescriptor_9af7607b46ac39b7, []int{18} } func (m *CachedResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1357,7 +1460,7 @@ func (m *Extent) Reset() { *m = Extent{} } func (m *Extent) String() string { return proto.CompactTextString(m) } func (*Extent) ProtoMessage() {} func (*Extent) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{20} + return fileDescriptor_9af7607b46ac39b7, []int{19} } func (m *Extent) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1425,7 +1528,7 @@ func (m *CachingOptions) Reset() { *m = CachingOptions{} } func (m *CachingOptions) String() string { return proto.CompactTextString(m) } func (*CachingOptions) ProtoMessage() {} func (*CachingOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{21} + return fileDescriptor_9af7607b46ac39b7, []int{20} } func (m *CachingOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1473,7 +1576,7 @@ func (m *Explanation) Reset() { *m = Explanation{} } func (m *Explanation) String() string { return proto.CompactTextString(m) } func (*Explanation) ProtoMessage() {} func (*Explanation) Descriptor() ([]byte, []int) { - return fileDescriptor_9af7607b46ac39b7, []int{22} + return fileDescriptor_9af7607b46ac39b7, []int{21} } func (m *Explanation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1528,7 +1631,8 @@ func init() { proto.RegisterType((*Vector)(nil), "queryrange.Vector") proto.RegisterType((*Matrix)(nil), "queryrange.Matrix") proto.RegisterType((*PrometheusResponseStats)(nil), "queryrange.PrometheusResponseStats") - proto.RegisterType((*PrometheusResponseSamplesStats)(nil), "queryrange.PrometheusResponseSamplesStats") + proto.RegisterType((*PrometheusResponseStats_Timings)(nil), "queryrange.PrometheusResponseStats.Timings") + proto.RegisterType((*PrometheusResponseStats_Samples)(nil), "queryrange.PrometheusResponseStats.Samples") proto.RegisterType((*PrometheusResponseQueryableSamplesStatsPerStep)(nil), "queryrange.PrometheusResponseQueryableSamplesStatsPerStep") proto.RegisterType((*SampleStream)(nil), "queryrange.SampleStream") proto.RegisterType((*Sample)(nil), "queryrange.Sample") @@ -1547,97 +1651,107 @@ func init() { } var fileDescriptor_9af7607b46ac39b7 = []byte{ - // 1427 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x73, 0x1b, 0xc5, - 0x12, 0xcf, 0x4a, 0xf2, 0x5a, 0x6a, 0xf9, 0xd9, 0xc9, 0x38, 0xef, 0x45, 0xf6, 0xf3, 0xf3, 0xea, - 0x6d, 0x28, 0xca, 0x84, 0x44, 0xaa, 0x32, 0x15, 0x0e, 0xa9, 0x22, 0xe0, 0x25, 0x01, 0x27, 0x95, - 0x0f, 0x67, 0x9c, 0xca, 0x81, 0x4b, 0x6a, 0x24, 0x0d, 0xd2, 0x92, 0xd5, 0xee, 0x66, 0x76, 0x36, - 0xd8, 0x37, 0xfe, 0x08, 0x0e, 0x9c, 0x28, 0xa8, 0xe2, 0x42, 0x15, 0x7f, 0x01, 0x37, 0x6e, 0x39, - 0x72, 0xe6, 0xb0, 0x50, 0x39, 0xee, 0x89, 0x3f, 0x81, 0x9a, 0x8f, 0xd5, 0xce, 0xca, 0x1f, 0x29, - 0x17, 0x17, 0xe0, 0x62, 0xcd, 0xf4, 0xd7, 0x74, 0xff, 0xba, 0xa7, 0xb7, 0xc7, 0x70, 0x65, 0x18, - 0x31, 0x4e, 0x0f, 0xfa, 0xcf, 0x53, 0xca, 0x7c, 0xca, 0xe4, 0xef, 0x21, 0x23, 0xe1, 0x98, 0x1a, - 0xcb, 0x5e, 0xcc, 0x22, 0x1e, 0x21, 0x28, 0x29, 0xeb, 0x17, 0xc7, 0xd1, 0x38, 0x92, 0xe4, 0xbe, - 0x58, 0x29, 0x89, 0xf5, 0xcd, 0x71, 0x14, 0x8d, 0x03, 0xda, 0x97, 0xbb, 0x41, 0xfa, 0x69, 0x7f, - 0x94, 0x32, 0xc2, 0xfd, 0x28, 0xd4, 0xfc, 0x0d, 0x7d, 0x9a, 0xfa, 0x89, 0x07, 0x7a, 0xa1, 0xb9, - 0x6b, 0xf3, 0xda, 0x24, 0x3c, 0x54, 0x2c, 0x77, 0x1f, 0x2e, 0xed, 0xb1, 0x68, 0x4a, 0xf9, 0x84, - 0xa6, 0x09, 0xa6, 0xcf, 0x53, 0x9a, 0xf0, 0x5d, 0x4a, 0x46, 0x94, 0xa1, 0x35, 0x68, 0x3c, 0x20, - 0x53, 0xda, 0xb1, 0xba, 0xd6, 0x56, 0xcb, 0x5b, 0xc8, 0x33, 0xc7, 0xba, 0x86, 0x25, 0x09, 0xfd, - 0x0f, 0xec, 0x27, 0x24, 0x48, 0x69, 0xd2, 0xa9, 0x75, 0xeb, 0x25, 0x53, 0x13, 0xdd, 0xac, 0x06, - 0x17, 0x8e, 0x58, 0x45, 0x08, 0x1a, 0x31, 0xe1, 0x13, 0x65, 0x0f, 0xcb, 0x35, 0xba, 0x08, 0x0b, - 0x09, 0x27, 0x8c, 0x77, 0x6a, 0x5d, 0x6b, 0xab, 0x8e, 0xd5, 0x06, 0x9d, 0x87, 0x3a, 0x0d, 0x47, - 0x9d, 0xba, 0xa4, 0x89, 0xa5, 0xd0, 0x4d, 0x38, 0x8d, 0x3b, 0x0d, 0x49, 0x92, 0x6b, 0xf4, 0x1e, - 0x2c, 0x72, 0x7f, 0x4a, 0xa3, 0x94, 0x77, 0x16, 0xba, 0xd6, 0x56, 0x7b, 0x7b, 0xad, 0xa7, 0xe2, - 0xec, 0x15, 0x71, 0xf6, 0x6e, 0x69, 0x94, 0xbc, 0xe6, 0xcb, 0xcc, 0x39, 0xf7, 0xd5, 0xaf, 0x8e, - 0x85, 0x0b, 0x1d, 0x71, 0xb4, 0x84, 0xbd, 0x63, 0x4b, 0x7f, 0xd4, 0x06, 0xed, 0xc2, 0xf2, 0x90, - 0x0c, 0x27, 0x7e, 0x38, 0x7e, 0x18, 0x0b, 0xcd, 0xa4, 0xb3, 0x28, 0x6d, 0xaf, 0xf7, 0x8c, 0xac, - 0x7d, 0x58, 0x91, 0xf0, 0x1a, 0xc2, 0x38, 0x9e, 0xd3, 0x43, 0xb7, 0x60, 0x51, 0x01, 0x99, 0x74, - 0x9a, 0xdd, 0xfa, 0x56, 0x7b, 0xfb, 0xb2, 0x69, 0xe2, 0x04, 0xd0, 0x0b, 0x24, 0x0b, 0x55, 0x0d, - 0x10, 0x4f, 0x3a, 0x2d, 0xe5, 0xa5, 0xdc, 0xb8, 0x8f, 0xa1, 0x63, 0x1a, 0x48, 0xe2, 0x28, 0x4c, - 0xe8, 0x9f, 0x4e, 0xdb, 0xf7, 0x35, 0x40, 0x47, 0xcd, 0x22, 0x17, 0xec, 0x7d, 0x4e, 0x78, 0x9a, - 0x68, 0x93, 0x90, 0x67, 0x8e, 0x9d, 0x48, 0x0a, 0xd6, 0x1c, 0xf4, 0x11, 0x34, 0x6e, 0x11, 0x4e, - 0x64, 0x1a, 0xe7, 0xc0, 0x2a, 0x2d, 0x0a, 0x09, 0xef, 0x3f, 0x02, 0xac, 0x3c, 0x73, 0x96, 0x47, - 0x84, 0x93, 0xab, 0xd1, 0xd4, 0xe7, 0x74, 0x1a, 0xf3, 0x43, 0x2c, 0xf5, 0xd1, 0x75, 0x68, 0xdd, - 0x66, 0x2c, 0x62, 0x8f, 0x0f, 0x63, 0x2a, 0xf3, 0xdf, 0xf2, 0x2e, 0xe5, 0x99, 0xb3, 0x4a, 0x0b, - 0xa2, 0xa1, 0x51, 0x4a, 0xa2, 0xb7, 0x60, 0x41, 0x6e, 0x64, 0x7d, 0xb4, 0xbc, 0xd5, 0x3c, 0x73, - 0x56, 0xa4, 0x8a, 0x21, 0xae, 0x24, 0xd0, 0xed, 0x32, 0x2d, 0x0b, 0x32, 0x2d, 0x6f, 0x9c, 0x94, - 0x16, 0x13, 0xd5, 0xf9, 0xbc, 0xb8, 0x5f, 0xd7, 0x60, 0xb9, 0x1a, 0x19, 0xea, 0x01, 0x60, 0x9a, - 0xa4, 0x01, 0x97, 0xce, 0x2b, 0xac, 0x96, 0xf3, 0xcc, 0x01, 0x36, 0xa3, 0x62, 0x43, 0x02, 0x7d, - 0x00, 0xb6, 0xda, 0xc9, 0x6c, 0xb4, 0xb7, 0x3b, 0xa6, 0x23, 0xfb, 0x64, 0x1a, 0x07, 0x74, 0x9f, - 0x33, 0x4a, 0xa6, 0xde, 0xb2, 0xc6, 0xcc, 0x56, 0x96, 0xb0, 0xd6, 0x43, 0x0f, 0x8a, 0xe2, 0xa8, - 0x4b, 0xd8, 0x2f, 0x9f, 0x1e, 0x89, 0x48, 0x55, 0xa2, 0xb0, 0x91, 0x5a, 0x26, 0x36, 0x92, 0x80, - 0xee, 0x42, 0xfb, 0xf6, 0x41, 0x1c, 0x90, 0x50, 0x5e, 0x1a, 0x09, 0x66, 0x7b, 0xfb, 0x92, 0x69, - 0xd5, 0x60, 0x7b, 0x2b, 0x79, 0xe6, 0xb4, 0x69, 0x49, 0xc0, 0xa6, 0xb2, 0xfb, 0x53, 0x0d, 0x36, - 0x4b, 0x1f, 0xee, 0x84, 0x09, 0x27, 0x21, 0x7f, 0x24, 0x2c, 0x9d, 0xa9, 0xb0, 0x70, 0xa5, 0xb0, - 0xde, 0x3c, 0x3e, 0x42, 0xd3, 0xfa, 0x3f, 0xa9, 0xc8, 0x7e, 0xac, 0xc1, 0xfa, 0xc9, 0x51, 0x9e, - 0xb9, 0xe0, 0xf6, 0x8c, 0x82, 0x13, 0x68, 0x6e, 0xbd, 0x1e, 0x4d, 0x25, 0xff, 0xb7, 0x2c, 0xc0, - 0xdf, 0x2d, 0xd8, 0x38, 0x2d, 0x28, 0x74, 0x05, 0xec, 0x64, 0x48, 0x02, 0xc2, 0x24, 0x74, 0xed, - 0xed, 0xf3, 0xbd, 0xe2, 0xeb, 0xa9, 0x6f, 0xdf, 0xee, 0x39, 0xac, 0x25, 0xd0, 0x4d, 0x58, 0x4a, - 0x38, 0xf3, 0xc3, 0xb1, 0xe2, 0x68, 0x00, 0xab, 0x37, 0xd6, 0xe0, 0xef, 0x9e, 0xc3, 0x15, 0x79, - 0x74, 0x15, 0xec, 0x17, 0x74, 0xc8, 0x23, 0xa6, 0x91, 0x42, 0xa6, 0xe6, 0x13, 0xc9, 0x11, 0xa7, - 0x29, 0x19, 0x21, 0x3d, 0x25, 0x9c, 0xf9, 0x07, 0x1a, 0x81, 0x8a, 0xf4, 0x7d, 0xc9, 0x11, 0xd2, - 0x4a, 0xc6, 0x6b, 0x82, 0x4e, 0x8b, 0xfb, 0x2e, 0xd8, 0x4f, 0x0a, 0x0b, 0x8b, 0x89, 0x3c, 0x59, - 0xdc, 0xad, 0xfa, 0xbc, 0x09, 0xe5, 0x14, 0x2e, 0x44, 0xdc, 0x5d, 0xb0, 0x95, 0x55, 0x74, 0x13, - 0xfe, 0x95, 0x18, 0x9d, 0xa7, 0xd0, 0x3e, 0xb1, 0x35, 0xe1, 0xaa, 0xb8, 0x1b, 0x54, 0xc7, 0x09, - 0x23, 0xef, 0xe8, 0x91, 0xe9, 0x92, 0x88, 0xea, 0xca, 0x6b, 0xaa, 0x45, 0x09, 0xab, 0xa2, 0x69, - 0xe7, 0x99, 0x53, 0xa8, 0x97, 0x7e, 0x7f, 0x59, 0xe9, 0x31, 0xc7, 0x29, 0xa2, 0x87, 0xf0, 0x6f, - 0x1e, 0x71, 0x12, 0xc8, 0xc4, 0x93, 0x41, 0x50, 0x70, 0xa5, 0x0f, 0x75, 0x6f, 0x2d, 0xcf, 0x9c, - 0xe3, 0x05, 0xf0, 0xf1, 0x64, 0xf4, 0x8d, 0x05, 0x1b, 0xc7, 0x72, 0xf6, 0x28, 0xdb, 0x17, 0x23, - 0x8a, 0x6a, 0xe6, 0x37, 0x4e, 0x0f, 0x6e, 0x5e, 0x59, 0x3a, 0xab, 0x2d, 0x78, 0xdd, 0x3c, 0x73, - 0x4e, 0x3d, 0x03, 0x9f, 0xca, 0x75, 0x7d, 0x38, 0xe3, 0x89, 0x62, 0xca, 0x78, 0x21, 0x66, 0x00, - 0x85, 0x0a, 0x56, 0x1b, 0xf4, 0x7f, 0x58, 0x12, 0xc3, 0x52, 0xc2, 0xc9, 0x34, 0x7e, 0x3a, 0x4d, - 0xf4, 0x8c, 0xd6, 0x9e, 0xd1, 0xee, 0x27, 0xee, 0xb7, 0x35, 0x58, 0x32, 0xeb, 0x01, 0x7d, 0x61, - 0x81, 0x1d, 0x90, 0x01, 0x0d, 0x8a, 0xd2, 0x59, 0x2d, 0x6f, 0xd5, 0x3d, 0x41, 0xdf, 0x23, 0x3e, - 0xf3, 0xf6, 0x45, 0x3f, 0xf9, 0x25, 0x73, 0x76, 0xc6, 0x3e, 0x9f, 0xa4, 0x83, 0xde, 0x30, 0x9a, - 0xf6, 0xf9, 0x84, 0x84, 0x51, 0x72, 0xcd, 0x8f, 0xf4, 0xaa, 0xef, 0x87, 0x9c, 0xb2, 0x90, 0x04, - 0xfd, 0xb9, 0xd9, 0x56, 0xd9, 0xd9, 0x19, 0x91, 0x98, 0x53, 0x26, 0x9a, 0xd2, 0x94, 0x72, 0xe6, - 0x0f, 0xb1, 0x3e, 0x17, 0xdd, 0x28, 0x0b, 0x4d, 0xe5, 0xe2, 0xc8, 0xc5, 0x2e, 0xfb, 0x99, 0x0c, - 0xb4, 0xac, 0x28, 0x84, 0x01, 0x26, 0x7e, 0xc2, 0xa3, 0x31, 0x13, 0xc5, 0x5f, 0x97, 0xea, 0xce, - 0xd1, 0xe2, 0xdf, 0x2d, 0x64, 0x64, 0x34, 0x17, 0xb4, 0xb5, 0xd6, 0x4c, 0x15, 0x1b, 0x56, 0xdc, - 0xef, 0x6a, 0x60, 0xeb, 0x36, 0xf0, 0x17, 0x40, 0xe7, 0x6d, 0x68, 0xab, 0x60, 0xe5, 0xd0, 0x27, - 0x73, 0x6a, 0x79, 0xad, 0x3c, 0x73, 0x54, 0xd2, 0xb1, 0xc9, 0x45, 0x1b, 0xd0, 0x9a, 0x65, 0x5b, - 0x8f, 0xe3, 0x25, 0x01, 0xdd, 0x83, 0x32, 0x62, 0xdd, 0xa9, 0xfe, 0x7b, 0x0a, 0x56, 0x12, 0x27, - 0xab, 0x8a, 0x53, 0xb9, 0x74, 0x3f, 0x86, 0x25, 0xb3, 0x85, 0x56, 0x6b, 0xb2, 0x75, 0x86, 0x9a, - 0xe4, 0xb0, 0x7a, 0x4c, 0x96, 0xaa, 0xb1, 0x58, 0xf3, 0xb1, 0xbc, 0x6f, 0xc6, 0x52, 0x7b, 0x7d, - 0x2c, 0x6a, 0xe6, 0x37, 0xdc, 0x8f, 0x61, 0x65, 0x4e, 0x46, 0x44, 0x30, 0x8c, 0xd2, 0x90, 0xcb, - 0xd3, 0x2c, 0xac, 0x36, 0xe2, 0x71, 0x93, 0xa4, 0xea, 0x0c, 0x0b, 0x8b, 0x25, 0xba, 0x0e, 0x8b, - 0x83, 0x74, 0xf8, 0x8c, 0xf2, 0xa2, 0xe2, 0x2a, 0x27, 0x97, 0x67, 0x4a, 0x19, 0x5c, 0xc8, 0xba, - 0x09, 0xac, 0xcc, 0xf1, 0xd0, 0x26, 0xc0, 0x20, 0x4a, 0xc3, 0x11, 0x61, 0xbe, 0x6e, 0x71, 0x0b, - 0xd8, 0xa0, 0x08, 0x8f, 0x82, 0xe8, 0x73, 0xca, 0xf4, 0xe9, 0x6a, 0x23, 0xa8, 0x69, 0x1c, 0x53, - 0xf5, 0x6d, 0xb2, 0xb0, 0xda, 0x94, 0xde, 0x37, 0x0c, 0xef, 0xdd, 0xcf, 0x60, 0x59, 0xbc, 0x7e, - 0xe8, 0x68, 0x36, 0xc5, 0xad, 0x41, 0xfd, 0x19, 0x3d, 0xd4, 0xe3, 0xc7, 0x62, 0x9e, 0x39, 0x62, - 0x8b, 0xc5, 0x1f, 0xf1, 0x42, 0xa3, 0x07, 0x9c, 0x86, 0xbc, 0xb8, 0x89, 0xa8, 0xfa, 0x29, 0x17, - 0x2c, 0x6f, 0x45, 0xdf, 0x9e, 0x42, 0x14, 0x17, 0x0b, 0xf7, 0x07, 0x0b, 0x6c, 0x25, 0x84, 0x9c, - 0xe2, 0x9d, 0xa8, 0xda, 0xb6, 0xac, 0x57, 0x49, 0x28, 0x9e, 0x8c, 0x6b, 0xea, 0xc9, 0x28, 0xcb, - 0x41, 0x79, 0x41, 0xc3, 0x91, 0x7a, 0x3b, 0x76, 0xa1, 0xc9, 0x19, 0x19, 0xd2, 0xa7, 0xfe, 0x48, - 0x8f, 0x6e, 0xc5, 0x9c, 0x25, 0xc9, 0x77, 0x46, 0xe8, 0x26, 0x34, 0x99, 0x0e, 0x47, 0x3f, 0x25, - 0x2f, 0x1e, 0x79, 0x4a, 0xee, 0x84, 0x87, 0xde, 0x52, 0x9e, 0x39, 0x33, 0x49, 0x3c, 0x5b, 0xdd, - 0x6d, 0x34, 0xeb, 0xe7, 0x1b, 0xee, 0x55, 0x05, 0x8d, 0xf1, 0x04, 0x5c, 0x87, 0xe6, 0xc8, 0x4f, - 0x44, 0xd3, 0x1d, 0x49, 0xc7, 0x9b, 0x78, 0xb6, 0x77, 0xc3, 0xca, 0xa8, 0x83, 0x36, 0xa0, 0x11, - 0x96, 0xaf, 0xb6, 0x66, 0x9e, 0x39, 0x72, 0x8f, 0xe5, 0x5f, 0xb4, 0x03, 0xcd, 0xe1, 0xc4, 0x0f, - 0x46, 0x8c, 0x86, 0x1a, 0xc9, 0x13, 0x87, 0x22, 0xe9, 0x63, 0x21, 0x8c, 0x67, 0x2b, 0xaf, 0xf3, - 0xf2, 0xd5, 0xa6, 0xf5, 0xf3, 0xab, 0x4d, 0xeb, 0xb7, 0x57, 0x9b, 0xd6, 0x27, 0xc6, 0x7f, 0x1c, - 0x06, 0xb6, 0x8c, 0xf1, 0x9d, 0x3f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x6b, 0x8e, 0x21, 0xfd, 0xb2, - 0x10, 0x00, 0x00, + // 1598 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x6f, 0x1b, 0xc7, + 0x15, 0xf7, 0x92, 0xd4, 0x92, 0x7c, 0x94, 0x25, 0x7b, 0x24, 0xd7, 0x94, 0xaa, 0x6a, 0xd5, 0x75, + 0x51, 0xa8, 0xfe, 0xa0, 0x50, 0x15, 0x6e, 0x01, 0x03, 0x75, 0xab, 0xad, 0xd5, 0xca, 0x86, 0x3f, + 0xe4, 0xa1, 0xe0, 0x43, 0x2f, 0xc6, 0x90, 0x9c, 0x52, 0x5b, 0x93, 0xbb, 0xeb, 0xd9, 0x59, 0x47, + 0xba, 0xe5, 0xcf, 0xc8, 0x29, 0x48, 0x80, 0x5c, 0x02, 0xe4, 0x90, 0x73, 0x6e, 0xb9, 0xf9, 0x98, + 0x73, 0x0e, 0x9b, 0xc0, 0xc7, 0x3d, 0x04, 0xb9, 0xe5, 0x1a, 0xcc, 0xc7, 0x72, 0x67, 0x29, 0x4a, + 0xb6, 0x90, 0x4b, 0x92, 0x0b, 0x39, 0xf3, 0xde, 0xef, 0xbd, 0x99, 0xf7, 0x7b, 0x6f, 0xbe, 0x16, + 0xae, 0xf7, 0x43, 0xc6, 0xe9, 0xd1, 0xd6, 0xcb, 0x84, 0x32, 0x9f, 0x32, 0xf9, 0x7f, 0xcc, 0x48, + 0x30, 0xa4, 0x46, 0xb3, 0x13, 0xb1, 0x90, 0x87, 0x08, 0x0a, 0xc9, 0xea, 0xf2, 0x30, 0x1c, 0x86, + 0x52, 0xbc, 0x25, 0x5a, 0x0a, 0xb1, 0xba, 0x3e, 0x0c, 0xc3, 0xe1, 0x88, 0x6e, 0xc9, 0x5e, 0x2f, + 0xf9, 0xdf, 0xd6, 0x20, 0x61, 0x84, 0xfb, 0x61, 0xa0, 0xf5, 0x6b, 0x7a, 0x34, 0xf5, 0x17, 0xf5, + 0x74, 0x43, 0x6b, 0x57, 0xa6, 0xad, 0x49, 0x70, 0xac, 0x54, 0x6e, 0x17, 0xae, 0xee, 0xb3, 0x70, + 0x4c, 0xf9, 0x21, 0x4d, 0x62, 0x4c, 0x5f, 0x26, 0x34, 0xe6, 0x7b, 0x94, 0x0c, 0x28, 0x43, 0x2b, + 0x50, 0x7b, 0x4c, 0xc6, 0xb4, 0x6d, 0x6d, 0x58, 0x9b, 0x4d, 0x6f, 0x2e, 0x4b, 0x1d, 0xeb, 0x16, + 0x96, 0x22, 0xf4, 0x3b, 0xb0, 0x9f, 0x91, 0x51, 0x42, 0xe3, 0x76, 0x65, 0xa3, 0x5a, 0x28, 0xb5, + 0xd0, 0x4d, 0x2b, 0x70, 0xf9, 0x84, 0x57, 0x84, 0xa0, 0x16, 0x11, 0x7e, 0xa8, 0xfc, 0x61, 0xd9, + 0x46, 0xcb, 0x30, 0x17, 0x73, 0xc2, 0x78, 0xbb, 0xb2, 0x61, 0x6d, 0x56, 0xb1, 0xea, 0xa0, 0x4b, + 0x50, 0xa5, 0xc1, 0xa0, 0x5d, 0x95, 0x32, 0xd1, 0x14, 0xb6, 0x31, 0xa7, 0x51, 0xbb, 0x26, 0x45, + 0xb2, 0x8d, 0xfe, 0x0e, 0x75, 0xee, 0x8f, 0x69, 0x98, 0xf0, 0xf6, 0xdc, 0x86, 0xb5, 0xd9, 0xda, + 0x5e, 0xe9, 0xa8, 0x38, 0x3b, 0x79, 0x9c, 0x9d, 0x7b, 0x9a, 0x25, 0xaf, 0xf1, 0x3a, 0x75, 0x2e, + 0x7c, 0xf0, 0x8d, 0x63, 0xe1, 0xdc, 0x46, 0x0c, 0x2d, 0x69, 0x6f, 0xdb, 0x72, 0x3e, 0xaa, 0x83, + 0xf6, 0x60, 0xa1, 0x4f, 0xfa, 0x87, 0x7e, 0x30, 0x7c, 0x12, 0x09, 0xcb, 0xb8, 0x5d, 0x97, 0xbe, + 0x57, 0x3b, 0x46, 0xd6, 0xfe, 0x55, 0x42, 0x78, 0x35, 0xe1, 0x1c, 0x4f, 0xd9, 0xa1, 0x7b, 0x50, + 0x57, 0x44, 0xc6, 0xed, 0xc6, 0x46, 0x75, 0xb3, 0xb5, 0x7d, 0xcd, 0x74, 0x71, 0x0a, 0xe9, 0x39, + 0x93, 0xb9, 0xa9, 0x26, 0x88, 0xc7, 0xed, 0xa6, 0x9a, 0xa5, 0xec, 0xb8, 0x07, 0xd0, 0x36, 0x1d, + 0xc4, 0x51, 0x18, 0xc4, 0xf4, 0x27, 0xa7, 0xed, 0xd3, 0x0a, 0xa0, 0x93, 0x6e, 0x91, 0x0b, 0x76, + 0x97, 0x13, 0x9e, 0xc4, 0xda, 0x25, 0x64, 0xa9, 0x63, 0xc7, 0x52, 0x82, 0xb5, 0x06, 0xfd, 0x1b, + 0x6a, 0xf7, 0x08, 0x27, 0x32, 0x8d, 0x53, 0x64, 0x15, 0x1e, 0x05, 0xc2, 0xfb, 0x8d, 0x20, 0x2b, + 0x4b, 0x9d, 0x85, 0x01, 0xe1, 0xe4, 0x66, 0x38, 0xf6, 0x39, 0x1d, 0x47, 0xfc, 0x18, 0x4b, 0x7b, + 0x74, 0x1b, 0x9a, 0xbb, 0x8c, 0x85, 0xec, 0xe0, 0x38, 0xa2, 0x32, 0xff, 0x4d, 0xef, 0x6a, 0x96, + 0x3a, 0x4b, 0x34, 0x17, 0x1a, 0x16, 0x05, 0x12, 0xfd, 0x09, 0xe6, 0x64, 0x47, 0xd6, 0x47, 0xd3, + 0x5b, 0xca, 0x52, 0x67, 0x51, 0x9a, 0x18, 0x70, 0x85, 0x40, 0xbb, 0x45, 0x5a, 0xe6, 0x64, 0x5a, + 0xfe, 0x70, 0x5a, 0x5a, 0x4c, 0x56, 0xa7, 0xf3, 0xe2, 0x7e, 0x58, 0x81, 0x85, 0x72, 0x64, 0xa8, + 0x03, 0x80, 0x69, 0x9c, 0x8c, 0xb8, 0x9c, 0xbc, 0xe2, 0x6a, 0x21, 0x4b, 0x1d, 0x60, 0x13, 0x29, + 0x36, 0x10, 0xe8, 0x9f, 0x60, 0xab, 0x9e, 0xcc, 0x46, 0x6b, 0xbb, 0x6d, 0x4e, 0xa4, 0x4b, 0xc6, + 0xd1, 0x88, 0x76, 0x39, 0xa3, 0x64, 0xec, 0x2d, 0x68, 0xce, 0x6c, 0xe5, 0x09, 0x6b, 0x3b, 0xf4, + 0x38, 0x2f, 0x8e, 0xaa, 0xa4, 0xfd, 0xda, 0xd9, 0x91, 0x88, 0x54, 0xc5, 0x8a, 0x1b, 0x69, 0x65, + 0x72, 0x23, 0x05, 0xe8, 0x01, 0xb4, 0x76, 0x8f, 0xa2, 0x11, 0x09, 0xe4, 0xa2, 0x91, 0x64, 0xb6, + 0xb6, 0xaf, 0x9a, 0x5e, 0x0d, 0xb5, 0xb7, 0x98, 0xa5, 0x4e, 0x8b, 0x16, 0x02, 0x6c, 0x1a, 0xbb, + 0x5f, 0x56, 0x60, 0xbd, 0x98, 0xc3, 0xfd, 0x20, 0xe6, 0x24, 0xe0, 0x4f, 0x85, 0xa7, 0x73, 0x15, + 0x16, 0x2e, 0x15, 0xd6, 0x1f, 0x67, 0x47, 0x68, 0x7a, 0xff, 0x35, 0x15, 0xd9, 0x17, 0x15, 0x58, + 0x3d, 0x3d, 0xca, 0x73, 0x17, 0xdc, 0xbe, 0x51, 0x70, 0x82, 0xcd, 0xcd, 0xb7, 0xb3, 0xa9, 0xf0, + 0xbf, 0xc8, 0x02, 0xfc, 0xde, 0x82, 0xb5, 0xb3, 0x82, 0x42, 0xd7, 0xc1, 0x8e, 0xfb, 0x64, 0x44, + 0x98, 0xa4, 0xae, 0xb5, 0x7d, 0xa9, 0x93, 0x9f, 0x9e, 0x7a, 0xf5, 0xed, 0x5d, 0xc0, 0x1a, 0x81, + 0xee, 0xc2, 0x7c, 0xcc, 0x99, 0x1f, 0x0c, 0x95, 0x46, 0x13, 0x58, 0x5e, 0xb1, 0x86, 0x7e, 0xef, + 0x02, 0x2e, 0xe1, 0xd1, 0x4d, 0xb0, 0x5f, 0xd1, 0x3e, 0x0f, 0x99, 0x66, 0x0a, 0x99, 0x96, 0xcf, + 0xa4, 0x46, 0x8c, 0xa6, 0x30, 0x02, 0x3d, 0x26, 0x9c, 0xf9, 0x47, 0x9a, 0x81, 0x12, 0xfa, 0x91, + 0xd4, 0x08, 0xb4, 0xc2, 0x78, 0x0d, 0xd0, 0x69, 0x71, 0xff, 0x0a, 0xf6, 0xb3, 0xdc, 0x43, 0x3d, + 0x96, 0x23, 0x8b, 0xb5, 0x55, 0x9d, 0x76, 0xa1, 0x26, 0x85, 0x73, 0x88, 0xbb, 0x07, 0xb6, 0xf2, + 0x8a, 0xee, 0xc2, 0xc5, 0xd8, 0xd8, 0x79, 0x72, 0xeb, 0x53, 0xb7, 0x26, 0x5c, 0x86, 0xbb, 0xdf, + 0xd9, 0xe5, 0xfb, 0x84, 0x91, 0x78, 0x84, 0xe5, 0x79, 0xed, 0x07, 0xc3, 0x58, 0x13, 0x7e, 0xe3, + 0x1d, 0xca, 0xa5, 0x73, 0xa0, 0x4c, 0xbc, 0x56, 0x96, 0x3a, 0xb9, 0x3d, 0xce, 0x1b, 0xc2, 0x67, + 0x1e, 0x67, 0xe5, 0xdd, 0x7d, 0xaa, 0x08, 0xb4, 0x4f, 0x6d, 0x3f, 0x61, 0x63, 0xf5, 0x87, 0x0a, + 0xd4, 0xf5, 0xa8, 0xe8, 0x6f, 0x70, 0x71, 0xf7, 0x15, 0x19, 0x1d, 0x84, 0x9c, 0x8c, 0x0e, 0x7c, + 0x7d, 0xaa, 0x56, 0xbc, 0xcb, 0x59, 0xea, 0x5c, 0xa4, 0xa6, 0x02, 0x97, 0x71, 0xe8, 0x0e, 0x2c, + 0xa8, 0x32, 0xeb, 0x86, 0x8c, 0x4b, 0xcb, 0x8a, 0xb4, 0x44, 0x62, 0x57, 0x62, 0x25, 0x0d, 0x9e, + 0x42, 0xa2, 0x87, 0xb0, 0x2c, 0xeb, 0x74, 0x9f, 0xd1, 0x88, 0xa8, 0x0b, 0x8c, 0xf4, 0x50, 0x95, + 0x1e, 0xda, 0x59, 0xea, 0x2c, 0xbf, 0x9c, 0xa1, 0xc7, 0x33, 0xad, 0x44, 0x08, 0xf7, 0x83, 0x80, + 0x32, 0x39, 0x3f, 0xe1, 0xa6, 0x56, 0x84, 0xe0, 0x9b, 0x0a, 0x5c, 0xc6, 0xc9, 0xd8, 0x8f, 0x68, + 0xff, 0x69, 0x42, 0x13, 0x2a, 0x0d, 0xe7, 0x8c, 0xd8, 0x4d, 0x05, 0x2e, 0xe3, 0x72, 0xc3, 0x82, + 0x34, 0xbb, 0x6c, 0x68, 0x92, 0x66, 0x76, 0x57, 0x3f, 0xaf, 0x40, 0x5d, 0xe7, 0x06, 0x3d, 0x81, + 0x2b, 0x52, 0x21, 0x63, 0x22, 0xbd, 0x11, 0xed, 0x4e, 0xea, 0xd9, 0xda, 0xac, 0x7a, 0x2b, 0x59, + 0xea, 0x5c, 0xe1, 0xb3, 0x00, 0x78, 0xb6, 0x1d, 0xfa, 0x33, 0xb4, 0xf6, 0x29, 0x79, 0xd1, 0x35, + 0xca, 0xa5, 0xaa, 0xb6, 0x90, 0xa8, 0x10, 0x63, 0x13, 0x83, 0x3e, 0xb2, 0x60, 0x6d, 0xe6, 0x18, + 0xfb, 0x94, 0x75, 0xc5, 0x75, 0xb4, 0x2a, 0x57, 0xc7, 0x9d, 0xb3, 0x6b, 0x6e, 0xda, 0x58, 0xd6, + 0xa0, 0xf6, 0xe0, 0x6d, 0x64, 0xa9, 0x73, 0xe6, 0x18, 0xf8, 0x4c, 0xad, 0xeb, 0xc3, 0x39, 0x47, + 0x14, 0x37, 0xca, 0x57, 0xe2, 0xbe, 0xa7, 0x88, 0xc4, 0xaa, 0x83, 0x7e, 0x0f, 0xf3, 0xe2, 0x62, + 0x1c, 0x73, 0x32, 0x8e, 0x9e, 0x8f, 0x35, 0x3d, 0xb8, 0x35, 0x91, 0x3d, 0x8a, 0xdd, 0x8f, 0x2b, + 0x30, 0x6f, 0xae, 0x7d, 0xf4, 0xbe, 0x05, 0xf6, 0x88, 0xf4, 0xe8, 0x28, 0xdf, 0x26, 0x96, 0x8a, + 0x1d, 0xf4, 0xa1, 0x90, 0xef, 0x13, 0x9f, 0x79, 0x5d, 0x71, 0x76, 0x7c, 0x9d, 0x3a, 0x3b, 0x43, + 0x9f, 0x1f, 0x26, 0xbd, 0x4e, 0x3f, 0x1c, 0x6f, 0xf1, 0x43, 0x12, 0x84, 0xf1, 0x2d, 0x3f, 0xd4, + 0xad, 0x2d, 0x3f, 0xe0, 0x94, 0x05, 0x64, 0xb4, 0x35, 0xf5, 0x8e, 0x51, 0x7e, 0x76, 0x06, 0x24, + 0xe2, 0x94, 0x89, 0x03, 0x68, 0x4c, 0x39, 0xf3, 0xfb, 0x58, 0x8f, 0x8b, 0xee, 0x98, 0xeb, 0xbf, + 0x3a, 0x6b, 0x13, 0x2f, 0xce, 0x2e, 0x19, 0x68, 0xb1, 0xce, 0x11, 0x06, 0x38, 0xf4, 0x63, 0x1e, + 0x0e, 0x99, 0xd8, 0xe8, 0x54, 0x2a, 0x9d, 0x93, 0x1b, 0xdd, 0x5e, 0x8e, 0x91, 0xd1, 0x5c, 0xd6, + 0xde, 0x9a, 0x13, 0x53, 0x6c, 0x78, 0x71, 0x3f, 0xa9, 0x80, 0xad, 0xb7, 0xfc, 0x9f, 0x01, 0x3b, + 0x37, 0xa0, 0xa5, 0x82, 0x95, 0x17, 0x7c, 0x99, 0x53, 0xcb, 0x6b, 0x66, 0xa9, 0xa3, 0x92, 0x8e, + 0x4d, 0x2d, 0x5a, 0x83, 0xe6, 0x24, 0xdb, 0xfa, 0xe9, 0x55, 0x08, 0xd0, 0x43, 0x28, 0x22, 0xd6, + 0xa7, 0xd2, 0x6f, 0xcf, 0xe0, 0x4a, 0xf2, 0x64, 0x95, 0x79, 0x2a, 0x9a, 0xee, 0x7f, 0x60, 0xde, + 0x3c, 0x2e, 0xcb, 0x35, 0xd9, 0x3c, 0x47, 0x4d, 0x72, 0x58, 0x9a, 0x91, 0xa5, 0x72, 0x2c, 0xd6, + 0x74, 0x2c, 0xff, 0x30, 0x63, 0xa9, 0xbc, 0x3d, 0x16, 0xf5, 0xbe, 0x33, 0xa6, 0x1f, 0xc1, 0xe2, + 0x14, 0x46, 0x44, 0xd0, 0x0f, 0x93, 0x80, 0xcb, 0xd1, 0x2c, 0xac, 0x3a, 0xe2, 0x21, 0x1b, 0x27, + 0x6a, 0x0c, 0x0b, 0x8b, 0x26, 0xba, 0x0d, 0xf5, 0x5e, 0xd2, 0x7f, 0x41, 0x79, 0x5e, 0x71, 0xa5, + 0x91, 0x8b, 0x31, 0x25, 0x06, 0xe7, 0x58, 0x37, 0x86, 0xc5, 0x29, 0x1d, 0x5a, 0x07, 0xe8, 0x85, + 0x49, 0x30, 0x20, 0xcc, 0xd7, 0xbb, 0xe2, 0x1c, 0x36, 0x24, 0x62, 0x46, 0xa3, 0xf0, 0x3d, 0xca, + 0xf4, 0xe8, 0xaa, 0x23, 0xa4, 0x49, 0x14, 0x51, 0x75, 0x0f, 0xb1, 0xb0, 0xea, 0x14, 0xb3, 0xaf, + 0x19, 0xb3, 0x77, 0xff, 0x0f, 0x0b, 0xe2, 0xa5, 0x4b, 0x07, 0x93, 0x1b, 0xfb, 0x0a, 0x54, 0x5f, + 0xd0, 0x63, 0x7d, 0xd5, 0xac, 0x67, 0xa9, 0x23, 0xba, 0x58, 0xfc, 0x88, 0xd7, 0x38, 0x3d, 0xe2, + 0x34, 0xe0, 0xf9, 0x4a, 0x44, 0xe5, 0x6b, 0x9b, 0x50, 0x79, 0x8b, 0x7a, 0xf5, 0xe4, 0x50, 0x9c, + 0x37, 0xdc, 0xcf, 0x2c, 0xb0, 0x15, 0x08, 0x39, 0xf9, 0x37, 0x01, 0xb5, 0xd3, 0xcb, 0x7a, 0x95, + 0x82, 0xfc, 0xf3, 0xc0, 0x8a, 0xfa, 0x3c, 0xa0, 0x76, 0x70, 0x39, 0x0b, 0x1a, 0x0c, 0xd4, 0x77, + 0x82, 0x0d, 0x68, 0x70, 0x46, 0xfa, 0xf4, 0xb9, 0x3f, 0xd0, 0xd7, 0xf4, 0xfc, 0x4e, 0x2d, 0xc5, + 0xf7, 0x07, 0xe8, 0x2e, 0x34, 0x98, 0x0e, 0x47, 0x7f, 0x36, 0x58, 0x3e, 0xf1, 0xd9, 0x60, 0x27, + 0x38, 0xf6, 0xe6, 0xb3, 0xd4, 0x99, 0x20, 0xf1, 0xa4, 0xf5, 0xa0, 0xd6, 0xa8, 0x5e, 0xaa, 0xb9, + 0x37, 0x15, 0x35, 0xc6, 0x73, 0x7f, 0x15, 0x1a, 0x03, 0x3f, 0x16, 0x9b, 0xee, 0x40, 0x4e, 0xbc, + 0x81, 0x27, 0x7d, 0x37, 0x28, 0x5d, 0x6b, 0xd1, 0x1a, 0xd4, 0x82, 0xe2, 0x85, 0xde, 0xc8, 0x52, + 0x47, 0xf6, 0xb1, 0xfc, 0x45, 0x3b, 0xd0, 0xe8, 0x1f, 0xfa, 0xa3, 0x01, 0xa3, 0x81, 0x66, 0xf2, + 0xd4, 0x0b, 0xb0, 0x9c, 0x63, 0x0e, 0xc6, 0x93, 0x96, 0xd7, 0x7e, 0xfd, 0x66, 0xdd, 0xfa, 0xea, + 0xcd, 0xba, 0xf5, 0xed, 0x9b, 0x75, 0xeb, 0xbf, 0xc6, 0xd7, 0xa5, 0x9e, 0x2d, 0x63, 0xfc, 0xcb, + 0x8f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xd4, 0xd6, 0xab, 0xc8, 0x9e, 0x12, 0x00, 0x00, } func (m *PrometheusRequestHeader) Marshal() (dAtA []byte, err error) { @@ -2341,12 +2455,24 @@ func (m *PrometheusResponseStats) MarshalToSizedBuffer(dAtA []byte) (int, error) i = encodeVarintQueryrange(dAtA, i, uint64(size)) } i-- + dAtA[i] = 0x12 + } + if m.Timings != nil { + { + size, err := m.Timings.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- dAtA[i] = 0xa } return len(dAtA) - i, nil } -func (m *PrometheusResponseSamplesStats) Marshal() (dAtA []byte, err error) { +func (m *PrometheusResponseStats_Timings) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -2356,12 +2482,75 @@ func (m *PrometheusResponseSamplesStats) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *PrometheusResponseSamplesStats) MarshalTo(dAtA []byte) (int, error) { +func (m *PrometheusResponseStats_Timings) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *PrometheusResponseSamplesStats) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *PrometheusResponseStats_Timings) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.ExecTotalTime != 0 { + i -= 4 + encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(math.Float32bits(float32(m.ExecTotalTime)))) + i-- + dAtA[i] = 0x35 + } + if m.ExecQueueTime != 0 { + i -= 4 + encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(math.Float32bits(float32(m.ExecQueueTime)))) + i-- + dAtA[i] = 0x2d + } + if m.InnerEvalTime != 0 { + i -= 4 + encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(math.Float32bits(float32(m.InnerEvalTime)))) + i-- + dAtA[i] = 0x25 + } + if m.QueryPreparationTime != 0 { + i -= 4 + encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(math.Float32bits(float32(m.QueryPreparationTime)))) + i-- + dAtA[i] = 0x1d + } + if m.ResultSortTime != 0 { + i -= 4 + encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(math.Float32bits(float32(m.ResultSortTime)))) + i-- + dAtA[i] = 0x15 + } + if m.EvalTotalTime != 0 { + i -= 4 + encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(math.Float32bits(float32(m.EvalTotalTime)))) + i-- + dAtA[i] = 0xd + } + return len(dAtA) - i, nil +} + +func (m *PrometheusResponseStats_Samples) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PrometheusResponseStats_Samples) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PrometheusResponseStats_Samples) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -2381,9 +2570,14 @@ func (m *PrometheusResponseSamplesStats) MarshalToSizedBuffer(dAtA []byte) (int, i = encodeVarintQueryrange(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x12 + dAtA[i] = 0x1a } } + if m.PeakSamples != 0 { + i = encodeVarintQueryrange(dAtA, i, uint64(m.PeakSamples)) + i-- + dAtA[i] = 0x10 + } if m.TotalQueryableSamples != 0 { i = encodeVarintQueryrange(dAtA, i, uint64(m.TotalQueryableSamples)) i-- @@ -3258,6 +3452,10 @@ func (m *PrometheusResponseStats) Size() (n int) { } var l int _ = l + if m.Timings != nil { + l = m.Timings.Size() + n += 1 + l + sovQueryrange(uint64(l)) + } if m.Samples != nil { l = m.Samples.Size() n += 1 + l + sovQueryrange(uint64(l)) @@ -3268,7 +3466,37 @@ func (m *PrometheusResponseStats) Size() (n int) { return n } -func (m *PrometheusResponseSamplesStats) Size() (n int) { +func (m *PrometheusResponseStats_Timings) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.EvalTotalTime != 0 { + n += 5 + } + if m.ResultSortTime != 0 { + n += 5 + } + if m.QueryPreparationTime != 0 { + n += 5 + } + if m.InnerEvalTime != 0 { + n += 5 + } + if m.ExecQueueTime != 0 { + n += 5 + } + if m.ExecTotalTime != 0 { + n += 5 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *PrometheusResponseStats_Samples) Size() (n int) { if m == nil { return 0 } @@ -3277,6 +3505,9 @@ func (m *PrometheusResponseSamplesStats) Size() (n int) { if m.TotalQueryableSamples != 0 { n += 1 + sovQueryrange(uint64(m.TotalQueryableSamples)) } + if m.PeakSamples != 0 { + n += 1 + sovQueryrange(uint64(m.PeakSamples)) + } if len(m.TotalQueryableSamplesPerStep) > 0 { for _, e := range m.TotalQueryableSamplesPerStep { l = e.Size() @@ -5270,6 +5501,42 @@ func (m *PrometheusResponseStats) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Timings", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Timings == nil { + m.Timings = &PrometheusResponseStats_Timings{} + } + if err := m.Timings.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Samples", wireType) } @@ -5299,7 +5566,7 @@ func (m *PrometheusResponseStats) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Samples == nil { - m.Samples = &PrometheusResponseSamplesStats{} + m.Samples = &PrometheusResponseStats_Samples{} } if err := m.Samples.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -5327,7 +5594,7 @@ func (m *PrometheusResponseStats) Unmarshal(dAtA []byte) error { } return nil } -func (m *PrometheusResponseSamplesStats) Unmarshal(dAtA []byte) error { +func (m *PrometheusResponseStats_Timings) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -5350,10 +5617,127 @@ func (m *PrometheusResponseSamplesStats) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: PrometheusResponseSamplesStats: wiretype end group for non-group") + return fmt.Errorf("proto: Timings: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: PrometheusResponseSamplesStats: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Timings: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field EvalTotalTime", wireType) + } + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.EvalTotalTime = float32(math.Float32frombits(v)) + case 2: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field ResultSortTime", wireType) + } + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.ResultSortTime = float32(math.Float32frombits(v)) + case 3: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field QueryPreparationTime", wireType) + } + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.QueryPreparationTime = float32(math.Float32frombits(v)) + case 4: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field InnerEvalTime", wireType) + } + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.InnerEvalTime = float32(math.Float32frombits(v)) + case 5: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field ExecQueueTime", wireType) + } + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.ExecQueueTime = float32(math.Float32frombits(v)) + case 6: + if wireType != 5 { + return fmt.Errorf("proto: wrong wireType = %d for field ExecTotalTime", wireType) + } + var v uint32 + if (iNdEx + 4) > l { + return io.ErrUnexpectedEOF + } + v = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:])) + iNdEx += 4 + m.ExecTotalTime = float32(math.Float32frombits(v)) + default: + iNdEx = preIndex + skippy, err := skipQueryrange(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthQueryrange + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PrometheusResponseStats_Samples) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Samples: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Samples: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -5376,6 +5760,25 @@ func (m *PrometheusResponseSamplesStats) Unmarshal(dAtA []byte) error { } } case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PeakSamples", wireType) + } + m.PeakSamples = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.PeakSamples |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TotalQueryableSamplesPerStep", wireType) } diff --git a/internal/cortex/querier/queryrange/queryrange.proto b/internal/cortex/querier/queryrange/queryrange.proto index 77e3bbdc55..e12fdd60f9 100644 --- a/internal/cortex/querier/queryrange/queryrange.proto +++ b/internal/cortex/querier/queryrange/queryrange.proto @@ -84,12 +84,22 @@ message Matrix { } message PrometheusResponseStats { - PrometheusResponseSamplesStats samples = 1 [(gogoproto.jsontag) = "samples"]; -} + message Timings { + float EvalTotalTime = 1 [(gogoproto.jsontag) = "evalTotalTime"]; + float ResultSortTime = 2 [(gogoproto.jsontag) = "resultSortTime"]; + float QueryPreparationTime = 3 [(gogoproto.jsontag) = "queryPreparationTime"]; + float InnerEvalTime = 4 [(gogoproto.jsontag) = "innerEvalTime"]; + float ExecQueueTime = 5 [(gogoproto.jsontag) = "execQueueTime"]; + float ExecTotalTime = 6 [(gogoproto.jsontag) = "execTotalTime"]; + } + message Samples { + int64 TotalQueryableSamples = 1 [(gogoproto.jsontag) = "totalQueryableSamples"]; + int64 PeakSamples = 2 [(gogoproto.jsontag) = "peakSamples"]; + repeated PrometheusResponseQueryableSamplesStatsPerStep totalQueryableSamplesPerStep = 3 [(gogoproto.jsontag) = "totalQueryableSamplesPerStep"]; + } -message PrometheusResponseSamplesStats { - int64 totalQueryableSamples = 1 [(gogoproto.jsontag) = "totalQueryableSamples"]; - repeated PrometheusResponseQueryableSamplesStatsPerStep totalQueryableSamplesPerStep = 2 [(gogoproto.jsontag) = "totalQueryableSamplesPerStep"]; + Timings timings = 1 [(gogoproto.jsontag) = "timings"]; + Samples samples = 2 [(gogoproto.jsontag) = "samples"]; } message PrometheusResponseQueryableSamplesStatsPerStep { diff --git a/internal/cortex/querier/queryrange/results_cache.go b/internal/cortex/querier/queryrange/results_cache.go index de0f7e8149..4e4c2f5a5d 100644 --- a/internal/cortex/querier/queryrange/results_cache.go +++ b/internal/cortex/querier/queryrange/results_cache.go @@ -49,9 +49,8 @@ type CacheGenNumberLoader interface { // ResultsCacheConfig is the config for the results cache. type ResultsCacheConfig struct { - CacheConfig cache.Config `yaml:"cache"` - Compression string `yaml:"compression"` - CacheQueryableSamplesStats bool `yaml:"cache_queryable_samples_stats"` + CacheConfig cache.Config `yaml:"cache"` + Compression string `yaml:"compression"` } // RegisterFlags registers flags. @@ -59,7 +58,6 @@ func (cfg *ResultsCacheConfig) RegisterFlags(f *flag.FlagSet) { cfg.CacheConfig.RegisterFlagsWithPrefix("frontend.", "", f) f.StringVar(&cfg.Compression, "frontend.compression", "", "Use compression in results cache. Supported values are: 'snappy' and '' (disable compression).") - f.BoolVar(&cfg.CacheQueryableSamplesStats, "frontend.cache-queryable-samples-stats", false, "Cache Statistics queryable samples on results cache.") } func (cfg *ResultsCacheConfig) Validate(qCfg querier.Config) error { @@ -70,10 +68,6 @@ func (cfg *ResultsCacheConfig) Validate(qCfg querier.Config) error { return errors.Errorf("unsupported compression type: %s", cfg.Compression) } - if cfg.CacheQueryableSamplesStats && !qCfg.EnablePerStepStats { - return errors.New("frontend.cache-queryable-samples-stats may only be enabled in conjunction with querier.per-step-stats-enabled. Please set the latter") - } - return cfg.CacheConfig.Validate() } @@ -159,12 +153,11 @@ type resultsCache struct { limits Limits splitter CacheSplitter - extractor Extractor - minCacheExtent int64 // discard any cache extent smaller than this - merger Merger - cacheGenNumberLoader CacheGenNumberLoader - shouldCache ShouldCacheFn - cacheQueryableSamplesStats bool + extractor Extractor + minCacheExtent int64 // discard any cache extent smaller than this + merger Merger + cacheGenNumberLoader CacheGenNumberLoader + shouldCache ShouldCacheFn } // NewResultsCacheMiddleware creates results cache middleware from config. @@ -198,36 +191,28 @@ func NewResultsCacheMiddleware( return MiddlewareFunc(func(next Handler) Handler { return &resultsCache{ - logger: logger, - cfg: cfg, - next: next, - cache: c, - limits: limits, - merger: merger, - extractor: extractor, - minCacheExtent: (5 * time.Minute).Milliseconds(), - splitter: splitter, - cacheGenNumberLoader: cacheGenNumberLoader, - shouldCache: shouldCache, - cacheQueryableSamplesStats: cfg.CacheQueryableSamplesStats, + logger: logger, + cfg: cfg, + next: next, + cache: c, + limits: limits, + merger: merger, + extractor: extractor, + minCacheExtent: (5 * time.Minute).Milliseconds(), + splitter: splitter, + cacheGenNumberLoader: cacheGenNumberLoader, + shouldCache: shouldCache, } }), c, nil } func (s resultsCache) Do(ctx context.Context, r Request) (Response, error) { tenantIDs, err := tenant.TenantIDs(ctx) - respWithStats := r.GetStats() != "" && s.cacheQueryableSamplesStats + respWithStats := r.GetStats() != "" if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } - // If cache_queryable_samples_stats is enabled we always need request the status upstream - if s.cacheQueryableSamplesStats { - r = r.WithStats("all") - } else { - r = r.WithStats("") - } - if s.shouldCache != nil && !s.shouldCache(r) { return s.next.Do(ctx, r) } @@ -447,6 +432,7 @@ func (s resultsCache) handleHit(ctx context.Context, r Request, extents []Extent if len(requests) == 0 { response, err := s.merger.MergeResponse(r, responses...) // No downstream requests so no need to write back to the cache. + response.AddHeader("X-Thanos-Results-Cache", fmt.Sprintf("hit; extents=%d", len(extents))) return response, nil, err } @@ -520,6 +506,8 @@ func (s resultsCache) handleHit(ctx context.Context, r Request, extents []Extent } response, err := s.merger.MergeResponse(r, responses...) + response.AddHeader("X-Thanos-Results-Cache", fmt.Sprintf("hit; extents=%d", len(extents))) + return response, mergedExtents, err } @@ -704,7 +692,12 @@ func extractStats(start, end int64, stats *PrometheusResponseStats) *PrometheusR return stats } - result := &PrometheusResponseStats{Samples: &PrometheusResponseSamplesStats{}} + result := &PrometheusResponseStats{ + Samples: &PrometheusResponseStats_Samples{ + PeakSamples: stats.Samples.PeakSamples, + }, + Timings: stats.Timings, + } for _, s := range stats.Samples.TotalQueryableSamplesPerStep { if start <= s.TimestampMs && s.TimestampMs <= end { result.Samples.TotalQueryableSamplesPerStep = append(result.Samples.TotalQueryableSamplesPerStep, s) diff --git a/pkg/queryfrontend/queryrange_codec.go b/pkg/queryfrontend/queryrange_codec.go index 36080d6821..fbe76408ab 100644 --- a/pkg/queryfrontend/queryrange_codec.go +++ b/pkg/queryfrontend/queryrange_codec.go @@ -128,6 +128,7 @@ func (c queryRangeCodec) DecodeRequest(_ context.Context, r *http.Request, forwa return nil, err } + result.Stats = r.FormValue("stats") result.Query = r.FormValue("query") result.Analyze = r.FormValue(queryv1.QueryAnalyzeParam) result.Engine = r.FormValue(queryv1.EngineParam) @@ -166,6 +167,7 @@ func (c queryRangeCodec) EncodeRequest(ctx context.Context, r queryrange.Request queryv1.DedupParam: []string{strconv.FormatBool(thanosReq.Dedup)}, queryv1.PartialResponseParam: []string{strconv.FormatBool(thanosReq.PartialResponse)}, queryv1.ReplicaLabelsParam: thanosReq.ReplicaLabels, + "stats": []string{thanosReq.Stats}, } if thanosReq.AutoDownsampling { diff --git a/pkg/queryfrontend/response.go b/pkg/queryfrontend/response.go index a6477130fa..4b808a73a9 100644 --- a/pkg/queryfrontend/response.go +++ b/pkg/queryfrontend/response.go @@ -64,3 +64,7 @@ func (m *ThanosLabelsResponse) GetStats() *queryrange.PrometheusResponseStats { func (m *ThanosSeriesResponse) GetStats() *queryrange.PrometheusResponseStats { return nil } + +func (m *ThanosSeriesResponse) AddHeader(key, value string) {} + +func (m *ThanosLabelsResponse) AddHeader(key, value string) {} diff --git a/pkg/queryfrontend/roundtrip.go b/pkg/queryfrontend/roundtrip.go index b901950ba8..e60eb08e37 100644 --- a/pkg/queryfrontend/roundtrip.go +++ b/pkg/queryfrontend/roundtrip.go @@ -78,6 +78,7 @@ func NewTripperware(config Config, reg prometheus.Registerer, logger log.Logger) queryInstantCodec, prometheus.WrapRegistererWith(prometheus.Labels{"tripperware": "query_instant"}, reg), config.ForwardHeaders, + logger, ) return func(next http.RoundTripper) http.RoundTripper { tripper := newRoundTripper( @@ -335,6 +336,7 @@ func newInstantQueryTripperware( codec queryrange.Codec, reg prometheus.Registerer, forwardHeaders []string, + logger log.Logger, ) queryrange.Tripperware { instantQueryMiddlewares := []queryrange.Middleware{} m := queryrange.NewInstrumentMiddlewareMetrics(reg)