Merge branch 'main' into fix-querier-address

clients/pkg/promtail/discovery/consulagent/consul.go

@@ -316,7 +316,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 		ticker := time.NewTicker(d.refreshInterval)
 
 		// Watched services and their cancellation functions.
-		services := make(map[string]func())
+		services := make(map[string]func(error))
 
 		for {
 			select {
@@ -340,7 +340,7 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) {
 // Watch the catalog for new services we would like to watch. This is called only
 // when we don't know yet the names of the services and need to ask Consul the
 // entire list of services.
-func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.Group, services map[string]func()) {
+func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.Group, services map[string]func(error)) {
 	agent := d.client.Agent()
 	level.Debug(d.logger).Log("msg", "Watching services", "tags", strings.Join(d.watchedTags, ","))
 
@@ -378,7 +378,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 			continue // We are already watching the service.
 		}
 
-		wctx, cancel := context.WithCancel(ctx)
+		wctx, cancel := context.WithCancelCause(ctx)
 		d.watchService(wctx, ch, name)
 		services[name] = cancel
 	}
@@ -390,7 +390,7 @@ func (d *Discovery) watchServices(ctx context.Context, ch chan<- []*targetgroup.
 				"msg", "removing service since consul no longer has a record of it",
 				"name", name)
 			// Call the watch cancellation function.
-			cancel()
+			cancel(errors.New("canceling service since consul no longer has a record of it"))
 			delete(services, name)
 
 			// Send clearing target group.

docs/sources/query/log_queries/_index.md

@@ -223,9 +223,9 @@ For example with `cluster="namespace"` the cluster is the label identifier, the
 We support multiple **value** types which are automatically inferred from the query input.
 
 - **String** is double quoted or backticked such as `"200"` or \``us-central1`\`.
-- **[Duration](https://golang.org/pkg/time/#ParseDuration)** is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+- **[Duration](https://golang.org/pkg/time/#ParseDuration)** is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The value of the label identifier used for comparison must be a string with a unit suffix to be parsed correctly, such as "0.10ms" or "1h30m". Optionally, `label_format` can be used to modify the value and append the unit before making the comparison.
 - **Number** are floating-point number (64bits), such as`250`, `89.923`.
-- **Bytes** is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "42MB", "1.5Kib" or "20b". Valid bytes units are "b", "kib", "kb", "mib", "mb", "gib",  "gb", "tib", "tb", "pib", "pb", "eib", "eb".
+- **Bytes** is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as "42MB", "1.5KiB" or "20B". Valid bytes units are "B", "kB", "MB", "GB", "TB", "KB", "KiB", "MiB", "GiB", "TiB".
 
 String type work exactly like Prometheus label matchers use in [log stream selector](#log-stream-selector). This means you can use the same operations (`=`,`!=`,`=~`,`!~`).
 
@@ -247,10 +247,10 @@ You can chain multiple predicates using `and` and `or` which respectively expres
 This means that all the following expressions are equivalent:
 
 ```logql
-| duration >= 20ms or size == 20kb and method!~"2.."
-| duration >= 20ms or size == 20kb | method!~"2.."
-| duration >= 20ms or size == 20kb , method!~"2.."
-| duration >= 20ms or size == 20kb  method!~"2.."
+| duration >= 20ms or size == 20KB and method!~"2.."
+| duration >= 20ms or size == 20KB | method!~"2.."
+| duration >= 20ms or size == 20KB , method!~"2.."
+| duration >= 20ms or size == 20KB  method!~"2.."
 
 ```
 

docs/sources/send-data/docker-driver/configuration.md

@@ -110,7 +110,7 @@ Stack name and service name for each swarm service and project name and service
 
 ## Labels
 
-Loki can received a set of labels along with log line. These labels are used to index log entries and query back logs using [LogQL stream selector]({{< relref "../../query/log_queries#log-stream-selector" >}}).
+Loki can receive a set of labels along with log line. These labels are used to index log entries and query back logs using [LogQL stream selector]({{< relref "../../query/log_queries#log-stream-selector" >}}).
 
 By default, the Docker driver will add the following labels to each log line:
 
@@ -128,7 +128,7 @@ next section for all supported options.
 ## Pipeline stages
 
 While you can provide `loki-pipeline-stage-file` it can be hard to mount the configuration file to the driver root filesystem.
-This is why another option `loki-pipeline-stages` is available allowing your to pass a list of stages inlined. Pipeline stages are run at last on every lines.
+This is why another option `loki-pipeline-stages` is available allowing you to pass a list of stages inlined. Pipeline stages are run at last on every lines.
 
 The example [docker-compose](https://github.com/grafana/loki/blob/main/clients/cmd/docker-driver/docker-compose.yaml) below configures 2 stages, one to extract level values and one to set it as a label:
 

docs/sources/send-data/k6/_index.md

@@ -9,8 +9,8 @@ weight:  900
 
 # Using k6 for load testing
 
-Grafana [k6](https://k6.io) is a modern load-testing tool.
-Its clean and approachable scripting [API](https://k6.io/docs/javascript-api/)
+Grafana [k6](https://grafana.com/oss/k6/) is a modern load-testing tool.
+Its clean and approachable scripting [API](https://grafana.com/docs/k6/latest/javascript-api/)
 works locally or in the cloud.
 Its configuration makes it flexible.
 
@@ -55,7 +55,7 @@ Use the custom-built k6 binary in the same way as a non-custom k6 binary:
 ```
 
 `test.js` is a Javascript load test.
-Refer to the [k6 documentation](https://k6.io/docs/) to get started.
+Refer to the [k6 documentation](https://grafana.com/docs/k6/latest/) to get started.
 
 ### Scripting API
 
@@ -75,7 +75,7 @@ Classes of this module are:
 | `Client` | client for writing and reading logs from Loki |
 
 `Config` and `Client` must be called on the k6 init context (see
-[Test life cycle](https://k6.io/docs/using-k6/test-life-cycle/)) outside of the
+[Test life cycle](https://grafana.com/docs/k6/latest/using-k6/test-lifecycle/)) outside of the
 default function so the client is only configured once and shared between all
 VU iterations.
 

docs/sources/send-data/k6/query-scenario.md

@@ -96,7 +96,7 @@ export default () => {
 ## Metrics
 
 The extension collects metrics that are printed in the
-[end-of-test summary](https://k6.io/docs/results-visualization/end-of-test-summary/) in addition to the built-in metrics.
+[end-of-test summary](https://grafana.com/docs/k6/latest/results-output/end-of-test/) in addition to the built-in metrics.
 These metrics are collected only for instant and range queries.
 
 | name                              | description                                  |

docs/sources/send-data/k6/write-scenario.md

@@ -59,17 +59,17 @@ These parameters can be adjusted in the load test:
 
 * The way to run k6
 
-    k6 supports three [execution modes](https://k6.io/docs/get-started/running-k6/#execution-modes) to run a test: local, distributed, and cloud.
+    k6 supports three [execution modes](https://grafana.com/docs/k6/latest/get-started/running-k6/#execution-modes) to run a test: local, distributed, and cloud.
     Whereas running your k6 load test from a single (local
     or remote) machine is easy to set up and fine for smaller Loki clusters,
     the single machine does not load test large Loki installations,
     because it cannot create the data to saturate the write path.
-    For larger tests, consider [these optimizations](https://k6.io/docs/testing-guides/running-large-tests/), or run them in [Grafana Cloud k6](/products/cloud/k6) or a Kubernetes cluster with the [k6 Operator](https://github.com/grafana/k6-operator).
+    For larger tests, consider [these optimizations](https://grafana.com/docs/k6/latest/testing-guides/running-large-tests/), or run them in [Grafana Cloud k6](/products/cloud/k6) or a Kubernetes cluster with the [k6 Operator](https://github.com/grafana/k6-operator).
 
 ## Metrics
 
 The extension collects two metrics that are printed in the
-[end-of-test summary](https://k6.io/docs/results-visualization/end-of-test-summary/) in addition to the built-in metrics.
+[end-of-test summary](https://grafana.com/docs/k6/latest/results-output/end-of-test/) in addition to the built-in metrics.
 
 | name | description |
 | ---- | ----------- |
@@ -80,7 +80,7 @@ The extension collects two metrics that are printed in the
 
 An HTTP request that successfully pushes logs to Loki
 responds with status `204 No Content`.
-The status code should be checked explicitly with a [k6 check](https://k6.io/docs/javascript-api/k6/check-val-sets-tags/).
+The status code should be checked explicitly with a [k6 check](https://grafana.com/docs/k6/latest/javascript-api/k6/check/).
 
 
 ## Javascript example

docs/sources/shared/configuration.md

@@ -377,6 +377,19 @@ bloom_build:
     # CLI flag: -bloom-build.builder.planner-address
     [planner_address: <string> | default = ""]
 
+    backoff_config:
+      # Minimum delay when backing off.
+      # CLI flag: -bloom-build.builder.backoff.backoff-min-period
+      [min_period: <duration> | default = 100ms]
+
+      # Maximum delay when backing off.
+      # CLI flag: -bloom-build.builder.backoff.backoff-max-period
+      [max_period: <duration> | default = 10s]
+
+      # Number of times to backoff and retry before failing.
+      # CLI flag: -bloom-build.builder.backoff.backoff-retries
+      [max_retries: <int> | default = 10]
+
 # Experimental: The bloom_gateway block configures the Loki bloom gateway
 # server, responsible for serving queries for filtering chunks based on filter
 # expressions.
@@ -3792,6 +3805,14 @@ These are values which allow you to control aspects of Loki's operation, most co
 # CLI flag: -operation-config.log-push-request-streams
 [log_push_request_streams: <boolean> | default = false]
 
+# Log metrics for duplicate lines received.
+# CLI flag: -operation-config.log-duplicate-metrics
+[log_duplicate_metrics: <boolean> | default = false]
+
+# Log stream info for duplicate lines received
+# CLI flag: -operation-config.log-duplicate-stream-info
+[log_duplicate_stream_info: <boolean> | default = false]
+
 # Log push errors with a rate limited logger, will show client push errors
 # without overly spamming logs.
 # CLI flag: -operation-config.limited-log-push-errors

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2
-	github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3
+	github.com/grafana/dskit v0.0.0-20240626184720-35810fdf1c6d
 	github.com/grafana/go-gelf/v2 v2.0.1
 	github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56
 	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd

go.sum

@@ -1017,8 +1017,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2 h1:qhugDMdQ4Vp68H0tp/0iN17DM2ehRo1rLEdOFe/gB8I=
 github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2/go.mod h1:w/aiO1POVIeXUQyl0VQSZjl5OAGDTL5aX+4v0RA1tcw=
-github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3 h1:k8vINlI4w+RYc37NRwQlRe/IHYoEbu6KAe2XdGDeV1U=
-github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3/go.mod h1:HvSf3uf8Ps2vPpzHeAFyZTdUcbVr+Rxpq1xcx7J/muc=
+github.com/grafana/dskit v0.0.0-20240626184720-35810fdf1c6d h1:CD8PWWX+9lYdgeMquSofmLErvCtk7jb+3/W/SH6oo/k=
+github.com/grafana/dskit v0.0.0-20240626184720-35810fdf1c6d/go.mod h1:HvSf3uf8Ps2vPpzHeAFyZTdUcbVr+Rxpq1xcx7J/muc=
 github.com/grafana/go-gelf/v2 v2.0.1 h1:BOChP0h/jLeD+7F9mL7tq10xVkDG15he3T1zHuQaWak=
 github.com/grafana/go-gelf/v2 v2.0.1/go.mod h1:lexHie0xzYGwCgiRGcvZ723bSNyNI8ZRD4s0CLobh90=
 github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 h1:xLuzPoOzdfNb/RF/IENCw+oLVdZB4G21VPhkHBgwSHY=