From a05431f879a8c29fac6356b6c46be62133c3e93c Mon Sep 17 00:00:00 2001 From: Callum Styan Date: Mon, 7 Oct 2024 10:01:28 -0700 Subject: [PATCH 1/6] fix: promtail config unmarshalling (#14408) Signed-off-by: Callum Styan --- clients/pkg/promtail/config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/pkg/promtail/config/config.go b/clients/pkg/promtail/config/config.go index 0454a8facf494..7e0e2b63fe173 100644 --- a/clients/pkg/promtail/config/config.go +++ b/clients/pkg/promtail/config/config.go @@ -42,7 +42,6 @@ type Config struct { // UnmarshalYAML implements the yaml.Unmarshaler interface. func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { - *c = Config{} // We want to set c to the defaults and then overwrite it with the input. // To make unmarshal fill the plain data struct rather than calling UnmarshalYAML // again, we have to hide it using a type indirection. From 1f3089282a2b4139e4b04a98ccb246f81b4a7cde Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:43:34 -0400 Subject: [PATCH 2/6] chore(deps): update dependency fluent-plugin-multi-format-parser to '~>1.1.0' (#14396) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- clients/cmd/fluentd/docker/Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/cmd/fluentd/docker/Gemfile b/clients/cmd/fluentd/docker/Gemfile index 981d6e701e63b..f52d506bc3bf9 100644 --- a/clients/cmd/fluentd/docker/Gemfile +++ b/clients/cmd/fluentd/docker/Gemfile @@ -3,4 +3,4 @@ source 'https://rubygems.org' gem 'fluentd', '1.15.3' -gem 'fluent-plugin-multi-format-parser', '~>1.0.0' +gem 'fluent-plugin-multi-format-parser', '~>1.1.0' From 91c7d344fe244471ba4eda2c33c6027663445371 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Tue, 8 Oct 2024 09:48:36 +0200 Subject: [PATCH 3/6] chore: Make metric for dequeued tasks in bloom-gateway a Histogram (#14413) This change allows to observe the distribution of how many tasks are dequeued at once over time. Signed-off-by: Christian Haudum --- pkg/bloomgateway/metrics.go | 9 +++++---- pkg/bloomgateway/worker.go | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/bloomgateway/metrics.go b/pkg/bloomgateway/metrics.go index 9fe096eec2ac4..4eeffbf8ad682 100644 --- a/pkg/bloomgateway/metrics.go +++ b/pkg/bloomgateway/metrics.go @@ -119,7 +119,7 @@ type workerMetrics struct { dequeueDuration *prometheus.HistogramVec queueDuration *prometheus.HistogramVec processDuration *prometheus.HistogramVec - tasksDequeued *prometheus.CounterVec + tasksDequeued *prometheus.HistogramVec tasksProcessed *prometheus.CounterVec blocksNotAvailable *prometheus.CounterVec blockQueryLatency *prometheus.HistogramVec @@ -147,11 +147,12 @@ func newWorkerMetrics(registerer prometheus.Registerer, namespace, subsystem str Name: "process_duration_seconds", Help: "Time spent processing tasks in seconds", }, append(labels, "status")), - tasksDequeued: r.NewCounterVec(prometheus.CounterOpts{ + tasksDequeued: r.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "tasks_dequeued_total", - Help: "Total amount of tasks that the worker dequeued from the queue", + Name: "tasks_dequeued", + Help: "Total amount of tasks that the worker dequeued from the queue at once", + Buckets: prometheus.ExponentialBuckets(1, 2, 8), // [1, 2, ..., 128] }, append(labels, "status")), tasksProcessed: r.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, diff --git a/pkg/bloomgateway/worker.go b/pkg/bloomgateway/worker.go index 6aa1082b89334..81092448ab52f 100644 --- a/pkg/bloomgateway/worker.go +++ b/pkg/bloomgateway/worker.go @@ -76,7 +76,7 @@ func (w *worker) running(_ context.Context) error { if err == queue.ErrStopped && len(items) == 0 { return err } - w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Inc() + w.metrics.tasksDequeued.WithLabelValues(w.id, labelFailure).Observe(1) level.Error(w.logger).Log("msg", "failed to dequeue tasks", "err", err, "items", len(items)) } idx = newIdx @@ -86,7 +86,7 @@ func (w *worker) running(_ context.Context) error { continue } - w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Add(float64(len(items))) + w.metrics.tasksDequeued.WithLabelValues(w.id, labelSuccess).Observe(float64(len(items))) tasks := make([]Task, 0, len(items)) for _, item := range items { From 5f325aac56e41848979e9e33a4a443e31ea525d0 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 8 Oct 2024 08:59:47 -0400 Subject: [PATCH 4/6] fix(storage/chunk/client/aws): have GetObject check for canceled context (#14420) --- .../chunk/client/aws/s3_storage_client.go | 2 +- .../client/aws/s3_storage_client_test.go | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/storage/chunk/client/aws/s3_storage_client.go b/pkg/storage/chunk/client/aws/s3_storage_client.go index 7747f27618008..9ab8c9116339f 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client.go @@ -393,7 +393,7 @@ func (a *S3ObjectClient) GetObject(ctx context.Context, objectKey string) (io.Re // Map the key into a bucket bucket := a.bucketFromKey(objectKey) - var lastErr error + lastErr := ctx.Err() retries := backoff.New(ctx, a.cfg.BackoffConfig) for retries.Ongoing() { diff --git a/pkg/storage/chunk/client/aws/s3_storage_client_test.go b/pkg/storage/chunk/client/aws/s3_storage_client_test.go index c582d9a4cab51..38b0215b79136 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client_test.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client_test.go @@ -234,6 +234,31 @@ func TestRequestMiddleware(t *testing.T) { } } +func TestS3ObjectClient_GetObject_CanceledContext(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, r.Header.Get("echo-me")) + })) + defer ts.Close() + + cfg := S3Config{ + Endpoint: ts.URL, + BucketNames: "buck-o", + S3ForcePathStyle: true, + Insecure: true, + AccessKeyID: "key", + SecretAccessKey: flagext.SecretWithValue("secret"), + } + + client, err := NewS3ObjectClient(cfg, hedging.Config{}) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, _, err = client.GetObject(ctx, "key") + require.Error(t, err, "GetObject should fail when given a canceled context") +} + func Test_Hedging(t *testing.T) { for _, tc := range []struct { name string From 8aa8a2bb0e766da4d64313d17337fa54ab84f8a4 Mon Sep 17 00:00:00 2001 From: benclive Date: Tue, 8 Oct 2024 15:05:10 +0100 Subject: [PATCH 5/6] fix(kafka): Set namespace for Loki kafka metrics (#14426) --- pkg/distributor/distributor.go | 20 ++++++++++++-------- pkg/kafka/writer_client.go | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 01dae3ee6e0cf..ac815cbe82cdb 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -279,11 +279,13 @@ func New( Help: "Total number of times the distributor has sharded streams", }), kafkaAppends: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ - Name: "kafka_appends_total", - Help: "The total number of appends sent to kafka ingest path.", + Namespace: constants.Loki, + Name: "distributor_kafka_appends_total", + Help: "The total number of appends sent to kafka ingest path.", }, []string{"partition", "status"}), kafkaWriteLatency: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Name: "kafka_latency_seconds", + Namespace: constants.Loki, + Name: "distributor_kafka_latency_seconds", Help: "Latency to write an incoming request to the ingest storage.", NativeHistogramBucketFactor: 1.1, NativeHistogramMinResetDuration: 1 * time.Hour, @@ -291,13 +293,15 @@ func New( Buckets: prometheus.DefBuckets, }), kafkaWriteBytesTotal: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "kafka_sent_bytes_total", - Help: "Total number of bytes sent to the ingest storage.", + Namespace: constants.Loki, + Name: "distributor_kafka_sent_bytes_total", + Help: "Total number of bytes sent to the ingest storage.", }), kafkaRecordsPerRequest: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Name: "kafka_records_per_write_request", - Help: "The number of records a single per-partition write request has been split into.", - Buckets: prometheus.ExponentialBuckets(1, 2, 8), + Namespace: constants.Loki, + Name: "distributor_kafka_records_per_write_request", + Help: "The number of records a single per-partition write request has been split into.", + Buckets: prometheus.ExponentialBuckets(1, 2, 8), }), writeFailuresManager: writefailures.NewManager(logger, registerer, cfg.WriteFailuresLogging, configs, "distributor"), kafkaWriter: kafkaWriter, diff --git a/pkg/kafka/writer_client.go b/pkg/kafka/writer_client.go index ddd12a646d692..59fefda31d19b 100644 --- a/pkg/kafka/writer_client.go +++ b/pkg/kafka/writer_client.go @@ -18,6 +18,8 @@ import ( "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "go.uber.org/atomic" + + "github.com/grafana/loki/v3/pkg/util/constants" ) // NewWriterClient returns the kgo.Client that should be used by the Writer. @@ -189,6 +191,7 @@ func NewProducer(client *kgo.Client, maxBufferedBytes int64, reg prometheus.Regi // Metrics. bufferedProduceBytes: promauto.With(reg).NewSummary( prometheus.SummaryOpts{ + Namespace: constants.Loki, Name: "buffered_produce_bytes", Help: "The buffered produce records in bytes. Quantile buckets keep track of buffered records size over the last 60s.", Objectives: map[float64]float64{0.5: 0.05, 0.99: 0.001, 1: 0.001}, @@ -197,16 +200,19 @@ func NewProducer(client *kgo.Client, maxBufferedBytes int64, reg prometheus.Regi }), bufferedProduceBytesLimit: promauto.With(reg).NewGauge( prometheus.GaugeOpts{ - Name: "buffered_produce_bytes_limit", - Help: "The bytes limit on buffered produce records. Produce requests fail once this limit is reached.", + Namespace: constants.Loki, + Name: "buffered_produce_bytes_limit", + Help: "The bytes limit on buffered produce records. Produce requests fail once this limit is reached.", }), produceRequestsTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "produce_requests_total", - Help: "Total number of produce requests issued to Kafka.", + Namespace: constants.Loki, + Name: "produce_requests_total", + Help: "Total number of produce requests issued to Kafka.", }), produceFailuresTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "produce_failures_total", - Help: "Total number of failed produce requests issued to Kafka.", + Namespace: constants.Loki, + Name: "produce_failures_total", + Help: "Total number of failed produce requests issued to Kafka.", }, []string{"reason"}), } From 8ec8cfb2c3546dd43b56dd2a603809c367c1d88f Mon Sep 17 00:00:00 2001 From: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com> Date: Tue, 8 Oct 2024 17:32:35 +0100 Subject: [PATCH 6/6] docs: Updated Promtail to Alloy (#14404) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/sources/setup/install/_index.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/sources/setup/install/_index.md b/docs/sources/setup/install/_index.md index 2b56cba78cb69..67356a44d4d91 100644 --- a/docs/sources/setup/install/_index.md +++ b/docs/sources/setup/install/_index.md @@ -9,7 +9,7 @@ weight: 200 # Install Loki -There are several methods of installing Loki and Promtail: +There are several methods of installing Loki: - [Install using Helm (recommended)]({{< relref "./helm" >}}) - [Install using Tanka]({{< relref "./tanka" >}}) @@ -17,12 +17,16 @@ There are several methods of installing Loki and Promtail: - [Install and run locally]({{< relref "./local" >}}) - [Install from source]({{< relref "./install-from-source" >}}) +Alloy: +- [Install Alloy](https://grafana.com/docs/alloy/latest/set-up/install/) +- [Ingest Logs with Alloy]({{< relref "../../send-data/alloy" >}}) + ## General process In order to run Loki, you must: -1. Download and install both Loki and Promtail. +1. Download and install both Loki and Alloy. 1. Download config files for both programs. 1. Start Loki. -1. Update the Promtail config file to get your logs into Loki. -1. Start Promtail. +1. Update the Alloy config file to get your logs into Loki. +1. Start Alloy.