Instrument querier inflight requests. (#11085)

**What this PR does / why we need it**:
#10858 removed the HTTP and gRPC server from the querier workers. That
means inflight requests are not instrumented anymore. This change adds a
middleware that does that.

This will add metrics such as 
```
# HELP cortex_inflight_requests Current number of inflight requests.
# TYPE cortex_inflight_requests gauge
cortex_inflight_requests{method="gRPC",route="*logproto.IndexStatsRequest"} 0
cortex_inflight_requests{method="gRPC",route="*queryrange.LabelRequest"} 0
cortex_inflight_requests{method="gRPC",route="*queryrange.LokiRequest"} 0
cortex_inflight_requests{method="gRPC",route="*queryrange.LokiSeriesRequest"} 0
```

**Checklist**
- [ ] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [ ] Tests updated
- [ ] `CHANGELOG.md` updated
- [ ] If the change is worth mentioning in the release notes, add
`add-to-release-notes` label
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/setup/upgrade/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](d10549e)
- [ ] If the change is deprecating or removing a configuration option,
update the `deprecated-config.yaml` and `deleted-config.yaml` files
respectively in the `tools/deprecated-config-checker` directory.
[Example
PR](0d4416a)

pkg/loki/modules.go

@@ -513,10 +513,20 @@ func (t *Loki) initQuerier() (services.Service, error) {
 	t.Server.HTTP.Path("/loki/api/v1/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler)))
 	t.Server.HTTP.Path("/api/prom/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler)))
 
+	internalHandler := queryrangebase.MergeMiddlewares(
+		serverutil.RecoveryMiddleware,
+		queryrange.Instrument{
+			QueryHandlerMetrics: queryrange.NewQueryHandlerMetrics(
+				prometheus.DefaultRegisterer,
+				t.Cfg.MetricsNamespace,
+			),
+		},
+	).Wrap(handler)
+
 	svc, err := querier.InitWorkerService(
 		querierWorkerServiceConfig,
 		prometheus.DefaultRegisterer,
-		serverutil.RecoveryMiddleware.Wrap(handler),
+		internalHandler,
 		t.Codec,
 	)
 	if err != nil {

pkg/querier/queryrange/instrument.go

@@ -0,0 +1,47 @@
+package queryrange
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
+)
+
+const (
+	gRPC = "gRPC"
+)
+
+type QueryHandlerMetrics struct {
+	InflightRequests *prometheus.GaugeVec
+}
+
+func NewQueryHandlerMetrics(registerer prometheus.Registerer, metricsNamespace string) *QueryHandlerMetrics {
+	return &QueryHandlerMetrics{
+		InflightRequests: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "inflight_requests",
+			Help:      "Current number of inflight requests.",
+		}, []string{"method", "route"}),
+	}
+}
+
+type Instrument struct {
+	*QueryHandlerMetrics
+}
+
+var _ queryrangebase.Middleware = Instrument{}
+
+// Wrap implements the queryrangebase.Middleware
+func (i Instrument) Wrap(next queryrangebase.Handler) queryrangebase.Handler {
+	return queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) {
+		route := fmt.Sprintf("%T", r)
+		inflight := i.InflightRequests.WithLabelValues(gRPC, route)
+		inflight.Inc()
+		defer inflight.Dec()
+
+		return next.Do(ctx, r)
+	})
+}

tools/dev/loki-boltdb-storage-s3/config/loki.yaml

@@ -123,7 +123,6 @@ storage_config:
         cache_location: /data/boltdb-cache
         index_gateway_client:
             server_address: index-gateway:9008
-        shared_store: s3
     aws:
       s3: s3://loki:supersecret@minio:9000/loki
       s3forcepathstyle: true