Instrument querier inflight requests.

pkg/loki/modules.go

@@ -513,10 +513,20 @@ func (t *Loki) initQuerier() (services.Service, error) {
 	t.Server.HTTP.Path("/loki/api/v1/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler)))
 	t.Server.HTTP.Path("/api/prom/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler)))
 
+	internalHandler := queryrangebase.MergeMiddlewares(
+		serverutil.RecoveryMiddleware,
+		queryrange.Instrument{
+			QueryHandlerMetrics: queryrange.NewQueryHandlerMetrics(
+				prometheus.DefaultRegisterer,
+				t.Cfg.MetricsNamespace,
+			),
+		},
+	).Wrap(handler)
+
 	svc, err := querier.InitWorkerService(
 		querierWorkerServiceConfig,
 		prometheus.DefaultRegisterer,
-		serverutil.RecoveryMiddleware.Wrap(handler),
+		internalHandler,
 		t.Codec,
 	)
 	if err != nil {

pkg/querier/queryrange/instrument.go

@@ -0,0 +1,47 @@
+package queryrange
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
+)
+
+const (
+	gRPC = "gRPC"
+)
+
+type QueryHandlerMetrics struct {
+	InflightRequests *prometheus.GaugeVec
+}
+
+func NewQueryHandlerMetrics(registerer prometheus.Registerer, metricsNamespace string) *QueryHandlerMetrics {
+	return &QueryHandlerMetrics{
+		InflightRequests: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "inflight_requests",
+			Help:      "Current number of inflight requests.",
+		}, []string{"method", "route"}),
+	}
+}
+
+type Instrument struct {
+	*QueryHandlerMetrics
+}
+
+var _ queryrangebase.Middleware = Instrument{}
+
+// Wrap implements the queryrangebase.Middleware
+func (i Instrument) Wrap(next queryrangebase.Handler) queryrangebase.Handler {
+	return queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) {
+		route := fmt.Sprintf("%T", r)
+		inflight := i.InflightRequests.WithLabelValues(gRPC, route)
+		inflight.Inc()
+		defer inflight.Dec()
+
+		return next.Do(ctx, r)
+	})
+}

tools/dev/loki-boltdb-storage-s3/config/loki.yaml

@@ -123,7 +123,6 @@ storage_config:
         cache_location: /data/boltdb-cache
         index_gateway_client:
             server_address: index-gateway:9008
-        shared_store: s3
     aws:
       s3: s3://loki:supersecret@minio:9000/loki
       s3forcepathstyle: true