From a629212ceba6bde476995c0550216f0431fb380b Mon Sep 17 00:00:00 2001 From: Ashwanth Date: Fri, 15 Nov 2024 19:39:33 +0530 Subject: [PATCH] feat(thanos): add new metric to track status codes (#14937) --- pkg/storage/bucket/azure/bucket_client.go | 8 ++-- pkg/storage/bucket/client.go | 47 +++++++++++++++++--- pkg/storage/bucket/gcs/bucket_client.go | 5 ++- pkg/storage/bucket/s3/bucket_client.go | 10 +++-- pkg/storage/bucket/sse_bucket_client_test.go | 2 +- pkg/storage/bucket/swift/bucket_client.go | 6 ++- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/pkg/storage/bucket/azure/bucket_client.go b/pkg/storage/bucket/azure/bucket_client.go index f7157cf1bca9a..487eb9f43e470 100644 --- a/pkg/storage/bucket/azure/bucket_client.go +++ b/pkg/storage/bucket/azure/bucket_client.go @@ -8,11 +8,11 @@ import ( "github.com/thanos-io/objstore/providers/azure" ) -func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { - return newBucketClient(cfg, name, logger, azure.NewBucketWithConfig) +func NewBucketClient(cfg Config, name string, logger log.Logger, wrapRT func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { + return newBucketClient(cfg, name, logger, wrapRT, azure.NewBucketWithConfig) } -func newBucketClient(cfg Config, name string, logger log.Logger, factory func(log.Logger, azure.Config, string, func(http.RoundTripper) http.RoundTripper) (*azure.Bucket, error)) (objstore.Bucket, error) { +func newBucketClient(cfg Config, name string, logger log.Logger, wrapRT func(http.RoundTripper) http.RoundTripper, factory func(log.Logger, azure.Config, string, func(http.RoundTripper) http.RoundTripper) (*azure.Bucket, error)) (objstore.Bucket, error) { // Start with default config to make sure that all parameters are set to sensible values, especially // HTTP Config field. bucketConfig := azure.DefaultConfig @@ -29,5 +29,5 @@ func newBucketClient(cfg Config, name string, logger log.Logger, factory func(lo bucketConfig.Endpoint = cfg.Endpoint } - return factory(logger, bucketConfig, name, nil) + return factory(logger, bucketConfig, name, wrapRT) } diff --git a/pkg/storage/bucket/client.go b/pkg/storage/bucket/client.go index 64338e8f02a18..88813b59eac30 100644 --- a/pkg/storage/bucket/client.go +++ b/pkg/storage/bucket/client.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "regexp" + "strconv" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -46,9 +47,24 @@ var ( ErrUnsupportedStorageBackend = errors.New("unsupported storage backend") ErrInvalidCharactersInStoragePrefix = errors.New("storage prefix contains invalid characters, it may only contain digits and English alphabet letters") - metrics = objstore.BucketMetrics(prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer), "") + metrics *objstore.Metrics + + // added to track the status codes by method + bucketRequestsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "loki", + Name: "objstore_bucket_transport_requests_total", + Help: "Total number of HTTP transport requests made to the bucket backend by status code and method.", + }, + []string{"status_code", "method"}, + ) ) +func init() { + prometheus.MustRegister(bucketRequestsTotal) + metrics = objstore.BucketMetrics(prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer), "") +} + // StorageBackendConfig holds configuration for accessing long-term storage. type StorageBackendConfig struct { // Backends @@ -176,13 +192,13 @@ func NewClient(ctx context.Context, backend string, cfg Config, name string, log // TODO: add support for other backends that loki already supports switch backend { case S3: - client, err = s3.NewBucketClient(cfg.S3, name, logger) + client, err = s3.NewBucketClient(cfg.S3, name, logger, instrumentTransport()) case GCS: - client, err = gcs.NewBucketClient(ctx, cfg.GCS, name, logger) + client, err = gcs.NewBucketClient(ctx, cfg.GCS, name, logger, instrumentTransport()) case Azure: - client, err = azure.NewBucketClient(cfg.Azure, name, logger) + client, err = azure.NewBucketClient(cfg.Azure, name, logger, instrumentTransport()) case Swift: - client, err = swift.NewBucketClient(cfg.Swift, name, logger) + client, err = swift.NewBucketClient(cfg.Swift, name, logger, instrumentTransport()) case Filesystem: client, err = filesystem.NewBucketClient(cfg.Filesystem) default: @@ -209,3 +225,24 @@ func NewClient(ctx context.Context, backend string, cfg Config, name string, log return instrumentedClient, nil } + +type instrumentedRoundTripper struct { + next http.RoundTripper +} + +func instrumentTransport() func(http.RoundTripper) http.RoundTripper { + return func(rt http.RoundTripper) http.RoundTripper { + return &instrumentedRoundTripper{next: rt} + } +} + +func (i *instrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := i.next.RoundTrip(req) + if err != nil { + return resp, err + } + + // Record status code and method metrics + bucketRequestsTotal.WithLabelValues(strconv.Itoa(resp.StatusCode), req.Method).Inc() + return resp, nil +} diff --git a/pkg/storage/bucket/gcs/bucket_client.go b/pkg/storage/bucket/gcs/bucket_client.go index 950202ea540e9..ea755a8517cad 100644 --- a/pkg/storage/bucket/gcs/bucket_client.go +++ b/pkg/storage/bucket/gcs/bucket_client.go @@ -2,6 +2,7 @@ package gcs import ( "context" + "net/http" "github.com/go-kit/log" "github.com/thanos-io/objstore" @@ -9,7 +10,7 @@ import ( ) // NewBucketClient creates a new GCS bucket client -func NewBucketClient(ctx context.Context, cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { +func NewBucketClient(ctx context.Context, cfg Config, name string, logger log.Logger, wrapRT func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { // start with default http configs bucketConfig := gcs.DefaultConfig bucketConfig.Bucket = cfg.BucketName @@ -18,5 +19,5 @@ func NewBucketClient(ctx context.Context, cfg Config, name string, logger log.Lo bucketConfig.MaxRetries = cfg.MaxRetries bucketConfig.HTTPConfig.Transport = cfg.Transport - return gcs.NewBucketWithConfig(ctx, logger, bucketConfig, name, nil) + return gcs.NewBucketWithConfig(ctx, logger, bucketConfig, name, wrapRT) } diff --git a/pkg/storage/bucket/s3/bucket_client.go b/pkg/storage/bucket/s3/bucket_client.go index 381f3436f53d4..a767e724cd630 100644 --- a/pkg/storage/bucket/s3/bucket_client.go +++ b/pkg/storage/bucket/s3/bucket_client.go @@ -1,6 +1,8 @@ package s3 import ( + "net/http" + "github.com/go-kit/log" "github.com/prometheus/common/model" "github.com/thanos-io/objstore" @@ -14,23 +16,23 @@ const ( ) // NewBucketClient creates a new S3 bucket client -func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) { +func NewBucketClient(cfg Config, name string, logger log.Logger, wrapRT func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { s3Cfg, err := newS3Config(cfg) if err != nil { return nil, err } - return s3.NewBucketWithConfig(logger, s3Cfg, name, nil) + return s3.NewBucketWithConfig(logger, s3Cfg, name, wrapRT) } // NewBucketReaderClient creates a new S3 bucket client -func NewBucketReaderClient(cfg Config, name string, logger log.Logger) (objstore.BucketReader, error) { +func NewBucketReaderClient(cfg Config, name string, logger log.Logger, wrapRT func(http.RoundTripper) http.RoundTripper) (objstore.BucketReader, error) { s3Cfg, err := newS3Config(cfg) if err != nil { return nil, err } - return s3.NewBucketWithConfig(logger, s3Cfg, name, nil) + return s3.NewBucketWithConfig(logger, s3Cfg, name, wrapRT) } func newS3Config(cfg Config) (s3.Config, error) { diff --git a/pkg/storage/bucket/sse_bucket_client_test.go b/pkg/storage/bucket/sse_bucket_client_test.go index 697e8837a2f32..5bba133fd4f86 100644 --- a/pkg/storage/bucket/sse_bucket_client_test.go +++ b/pkg/storage/bucket/sse_bucket_client_test.go @@ -56,7 +56,7 @@ func TestSSEBucketClient_Upload_ShouldInjectCustomSSEConfig(t *testing.T) { Insecure: true, } - s3Client, err := s3.NewBucketClient(s3Cfg, "test", log.NewNopLogger()) + s3Client, err := s3.NewBucketClient(s3Cfg, "test", log.NewNopLogger(), nil) require.NoError(t, err) // Configure the config provider with NO KMS key ID. diff --git a/pkg/storage/bucket/swift/bucket_client.go b/pkg/storage/bucket/swift/bucket_client.go index 28f3c922c4254..93ecdee9da7f7 100644 --- a/pkg/storage/bucket/swift/bucket_client.go +++ b/pkg/storage/bucket/swift/bucket_client.go @@ -1,6 +1,8 @@ package swift import ( + "net/http" + "github.com/go-kit/log" "github.com/prometheus/common/model" "github.com/thanos-io/objstore" @@ -9,7 +11,7 @@ import ( ) // NewBucketClient creates a new Swift bucket client -func NewBucketClient(cfg Config, _ string, logger log.Logger) (objstore.Bucket, error) { +func NewBucketClient(cfg Config, _ string, logger log.Logger, wrapper func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) { bucketConfig := swift.Config{ AuthVersion: cfg.AuthVersion, AuthUrl: cfg.AuthURL, @@ -37,5 +39,5 @@ func NewBucketClient(cfg Config, _ string, logger log.Logger) (objstore.Bucket, } bucketConfig.HTTPConfig.Transport = cfg.Transport - return swift.NewContainerFromConfig(logger, &bucketConfig, false, nil) + return swift.NewContainerFromConfig(logger, &bucketConfig, false, wrapper) }