From 10bbd8473cd9c5f76cb92cd4897a67ac92c73b6c Mon Sep 17 00:00:00 2001
From: Karsten Jeschkies <karsten.jeschkies@grafana.com>
Date: Mon, 30 Oct 2023 17:28:37 +0100
Subject: [PATCH 1/2] Instrument querier inflight requests.

---
 pkg/loki/modules.go               | 7 ++++++-
 pkg/querier/queryrange/metrics.go | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index 7c67f5e3df708..c33278c014f90 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -513,10 +513,15 @@ func (t *Loki) initQuerier() (services.Service, error) {
 	t.Server.HTTP.Path("/loki/api/v1/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler)))
 	t.Server.HTTP.Path("/api/prom/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler)))
 
+	internalHandler := queryrangebase.MergeMiddlewares(
+		serverutil.RecoveryMiddleware,
+		queryrange.Instrument{},
+	).Wrap(handler)
+
 	svc, err := querier.InitWorkerService(
 		querierWorkerServiceConfig,
 		prometheus.DefaultRegisterer,
-		serverutil.RecoveryMiddleware.Wrap(handler),
+		internalHandler,
 		t.Codec,
 	)
 	if err != nil {
diff --git a/pkg/querier/queryrange/metrics.go b/pkg/querier/queryrange/metrics.go
index 390f2c81d771e..0917c44de2be1 100644
--- a/pkg/querier/queryrange/metrics.go
+++ b/pkg/querier/queryrange/metrics.go
@@ -14,6 +14,7 @@ type Metrics struct {
 	*SplitByMetrics
 	*LogResultCacheMetrics
 	*queryrangebase.ResultsCacheMetrics
+	*QueryHandlerMetrics
 }
 
 type MiddlewareMapperMetrics struct {
@@ -36,5 +37,6 @@ func NewMetrics(registerer prometheus.Registerer, metricsNamespace string) *Metr
 		SplitByMetrics:              NewSplitByMetrics(registerer),
 		LogResultCacheMetrics:       NewLogResultCacheMetrics(registerer),
 		ResultsCacheMetrics:         queryrangebase.NewResultsCacheMetrics(registerer),
+		QueryHandlerMetrics:         NewQueryHandlerMetrics(registerer, metricsNamespace),
 	}
 }

From 2fdace3b7776471856978238b9b81cd27f33f9e6 Mon Sep 17 00:00:00 2001
From: Karsten Jeschkies <karsten.jeschkies@grafana.com>
Date: Mon, 30 Oct 2023 18:31:15 +0100
Subject: [PATCH 2/2] Wire up metrics.

---
 pkg/loki/modules.go                           |  7 ++-
 pkg/querier/queryrange/instrument.go          | 47 +++++++++++++++++++
 pkg/querier/queryrange/metrics.go             |  2 -
 .../loki-boltdb-storage-s3/config/loki.yaml   |  1 -
 4 files changed, 53 insertions(+), 4 deletions(-)
 create mode 100644 pkg/querier/queryrange/instrument.go

diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index c33278c014f90..fb9d32cfdde3b 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -515,7 +515,12 @@ func (t *Loki) initQuerier() (services.Service, error) {
 
 	internalHandler := queryrangebase.MergeMiddlewares(
 		serverutil.RecoveryMiddleware,
-		queryrange.Instrument{},
+		queryrange.Instrument{
+			QueryHandlerMetrics: queryrange.NewQueryHandlerMetrics(
+				prometheus.DefaultRegisterer,
+				t.Cfg.MetricsNamespace,
+			),
+		},
 	).Wrap(handler)
 
 	svc, err := querier.InitWorkerService(
diff --git a/pkg/querier/queryrange/instrument.go b/pkg/querier/queryrange/instrument.go
new file mode 100644
index 0000000000000..577bb1fd1f254
--- /dev/null
+++ b/pkg/querier/queryrange/instrument.go
@@ -0,0 +1,47 @@
+package queryrange
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
+)
+
+const (
+	gRPC = "gRPC"
+)
+
+type QueryHandlerMetrics struct {
+	InflightRequests *prometheus.GaugeVec
+}
+
+func NewQueryHandlerMetrics(registerer prometheus.Registerer, metricsNamespace string) *QueryHandlerMetrics {
+	return &QueryHandlerMetrics{
+		InflightRequests: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: metricsNamespace,
+			Name:      "inflight_requests",
+			Help:      "Current number of inflight requests.",
+		}, []string{"method", "route"}),
+	}
+}
+
+type Instrument struct {
+	*QueryHandlerMetrics
+}
+
+var _ queryrangebase.Middleware = Instrument{}
+
+// Wrap implements the queryrangebase.Middleware
+func (i Instrument) Wrap(next queryrangebase.Handler) queryrangebase.Handler {
+	return queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) {
+		route := fmt.Sprintf("%T", r)
+		inflight := i.InflightRequests.WithLabelValues(gRPC, route)
+		inflight.Inc()
+		defer inflight.Dec()
+
+		return next.Do(ctx, r)
+	})
+}
diff --git a/pkg/querier/queryrange/metrics.go b/pkg/querier/queryrange/metrics.go
index 0917c44de2be1..390f2c81d771e 100644
--- a/pkg/querier/queryrange/metrics.go
+++ b/pkg/querier/queryrange/metrics.go
@@ -14,7 +14,6 @@ type Metrics struct {
 	*SplitByMetrics
 	*LogResultCacheMetrics
 	*queryrangebase.ResultsCacheMetrics
-	*QueryHandlerMetrics
 }
 
 type MiddlewareMapperMetrics struct {
@@ -37,6 +36,5 @@ func NewMetrics(registerer prometheus.Registerer, metricsNamespace string) *Metr
 		SplitByMetrics:              NewSplitByMetrics(registerer),
 		LogResultCacheMetrics:       NewLogResultCacheMetrics(registerer),
 		ResultsCacheMetrics:         queryrangebase.NewResultsCacheMetrics(registerer),
-		QueryHandlerMetrics:         NewQueryHandlerMetrics(registerer, metricsNamespace),
 	}
 }
diff --git a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
index 48161f4e7c75e..4a0c307073dfd 100644
--- a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
+++ b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
@@ -123,7 +123,6 @@ storage_config:
         cache_location: /data/boltdb-cache
         index_gateway_client:
             server_address: index-gateway:9008
-        shared_store: s3
     aws:
       s3: s3://loki:supersecret@minio:9000/loki
       s3forcepathstyle: true