diff --git a/.drone/drone.yml b/.drone/drone.yml index ea3c04d46ef9d..d2be4bdb369b8 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -139,7 +139,7 @@ steps: depends_on: - clone environment: {} - image: grafana/loki-build-image:0.33.4 + image: grafana/loki-build-image:0.33.5 name: documentation-helm-reference-check trigger: ref: @@ -1085,7 +1085,7 @@ steps: from_secret: docker_password DOCKER_USERNAME: from_secret: docker_username - image: grafana/loki-build-image:0.33.4 + image: grafana/loki-build-image:0.33.5 name: build and push privileged: true volumes: @@ -1308,6 +1308,6 @@ kind: secret name: gpg_private_key --- kind: signature -hmac: 335170654951c8fdd9cb1b96b4290febb74b86ebab07cfe65d680299faa767bf +hmac: 8a2db8460244184bb92d99cfe9a366e0a1cce91034cfe3784436a2f178b976c7 ... diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 541e0da230904..12299b8cfbab5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -2,7 +2,7 @@ "check": "uses": "grafana/loki-release/.github/workflows/check.yml@main" "with": - "build_image": "grafana/loki-build-image:0.33.4" + "build_image": "grafana/loki-build-image:0.33.5" "golang_ci_lint_version": "v1.55.1" "release_lib_ref": "main" "skip_validation": false diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index 1f89cdc942a64..e6527ef1250d0 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -16,7 +16,7 @@ jobs: check: uses: "grafana/loki-release/.github/workflows/check.yml@main" with: - build_image: "grafana/loki-build-image:0.33.4" + build_image: "grafana/loki-build-image:0.33.5" golang_ci_lint_version: "v1.55.1" release_lib_ref: "main" skip_validation: false @@ -141,7 +141,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.33.4" + --entrypoint /bin/sh "grafana/loki-build-image:0.33.5" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 36748f8c7cae9..0461ce62c3eb9 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -16,7 +16,7 @@ jobs: check: uses: "grafana/loki-release/.github/workflows/check.yml@main" with: - build_image: "grafana/loki-build-image:0.33.4" + build_image: "grafana/loki-build-image:0.33.5" golang_ci_lint_version: "v1.55.1" release_lib_ref: "main" skip_validation: false @@ -141,7 +141,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.33.4" + --entrypoint /bin/sh "grafana/loki-build-image:0.33.5" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages diff --git a/Makefile b/Makefile index cadce23fe2624..ccca8ba5ee3b8 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) BUILD_IN_CONTAINER ?= true # ensure you run `make drone` and `make release-workflows` after changing this -BUILD_IMAGE_VERSION ?= 0.33.4 +BUILD_IMAGE_VERSION ?= 0.33.5 GO_VERSION := 1.22.5 # Docker image info diff --git a/docs/sources/alert/_index.md b/docs/sources/alert/_index.md index 1d99e56d1a560..e12e073c3b889 100644 --- a/docs/sources/alert/_index.md +++ b/docs/sources/alert/_index.md @@ -179,7 +179,7 @@ The Ruler's Prometheus compatibility further accentuates the marriage between me ### Black box monitoring -We don't always control the source code of applications we run. Load balancers and a myriad of other components, both open source and closed third-party, support our applications while they don't expose the metrics we want. Some don't expose any metrics at all. Loki's alerting and recording rules can produce metrics and alert on the state of the system, bringing the components into our observability stack by using the logs. This is an incredibly powerful way to introduce advanced observability into legacy architectures. +We don't always control the source code of applications we run. Load balancers and a myriad of other components, both open source and closed third-party, support our applications while they don't expose the metrics we want. Some don't expose any metrics at all. The Loki alerting and recording rules can produce metrics and alert on the state of the system, bringing the components into our observability stack by using the logs. This is an incredibly powerful way to introduce advanced observability into legacy architectures. ### Event alerting diff --git a/docs/sources/configure/storage.md b/docs/sources/configure/storage.md index a815b98f98897..27466dbc6e50c 100644 --- a/docs/sources/configure/storage.md +++ b/docs/sources/configure/storage.md @@ -27,7 +27,7 @@ You can find more detailed information about all of the storage options in the [ ## Single Store -Single Store refers to using object storage as the storage medium for both Loki's index as well as its data ("chunks"). There are two supported modes: +Single Store refers to using object storage as the storage medium for both the Loki index as well as its data ("chunks"). There are two supported modes: ### TSDB (recommended) @@ -83,7 +83,7 @@ You may use any substitutable services, such as those that implement the S3 API ### Cassandra (deprecated) -Cassandra is a popular database and one of Loki's possible chunk stores and is production safe. +Cassandra is a popular database and one of the possible chunk stores for Loki and is production safe. {{< collapse title="Title of hidden content" >}} This storage type for chunks is deprecated and may be removed in future major versions of Loki. diff --git a/docs/sources/get-started/_index.md b/docs/sources/get-started/_index.md index 60bbfb3e531f3..e4c7c54160295 100644 --- a/docs/sources/get-started/_index.md +++ b/docs/sources/get-started/_index.md @@ -9,7 +9,7 @@ description: Provides an overview of the steps for implementing Grafana Loki to {{< youtube id="1uk8LtQqsZQ" >}} -Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream. +Loki is a horizontally scalable, highly available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream. Because all Loki implementations are unique, the installation process is different for every customer. But there are some steps in the process that @@ -26,13 +26,13 @@ To collect logs and view your log data generally involves the following steps: 1. Deploy [Grafana Alloy](https://grafana.com/docs/alloy/latest/) to collect logs from your applications. 1. On Kubernetes, deploy the Grafana Flow using the Helm chart. Configure Grafana Alloy to scrape logs from your Kubernetes cluster, and add your Loki endpoint details. See the following section for an example Grafana Alloy configuration file. 1. Add [labels](https://grafana.com/docs/loki//get-started/labels/) to your logs following our [best practices](https://grafana.com/docs/loki//get-started/labels/bp-labels/). Most Loki users start by adding labels which describe where the logs are coming from (region, cluster, environment, etc.). -1. Deploy [Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/) or [Grafana Cloud](https://grafana.com/docs/grafana-cloud/quickstart/) and configure a [Loki datasource](https://grafana.com/docs/grafana/latest/datasources/loki/configure-loki-data-source/). +1. Deploy [Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/) or [Grafana Cloud](https://grafana.com/docs/grafana-cloud/quickstart/) and configure a [Loki data source](https://grafana.com/docs/grafana/latest/datasources/loki/configure-loki-data-source/). 1. Select the [Explore feature](https://grafana.com/docs/grafana/latest/explore/) in the Grafana main menu. To [view logs in Explore](https://grafana.com/docs/grafana/latest/explore/logs-integration/): 1. Pick a time range. - 1. Choose the Loki datasource. + 1. Choose the Loki data source. 1. Use [LogQL](https://grafana.com/docs/loki//query/) in the [query editor](https://grafana.com/docs/grafana/latest/datasources/loki/query-editor/), use the Builder view to explore your labels, or select from sample pre-configured queries using the **Kick start your query** button. -**Next steps:** Learn more about Loki’s query language, [LogQL](https://grafana.com/docs/loki//query/). +**Next steps:** Learn more about the Loki query language, [LogQL](https://grafana.com/docs/loki//query/). ## Example Grafana Alloy and Agent configuration files to ship Kubernetes Pod logs to Loki diff --git a/docs/sources/get-started/architecture.md b/docs/sources/get-started/architecture.md index 9caeb717144bd..42b81232b9886 100644 --- a/docs/sources/get-started/architecture.md +++ b/docs/sources/get-started/architecture.md @@ -1,7 +1,7 @@ --- title: Loki architecture menutitle: Architecture -description: Describes Grafana Loki's architecture. +description: Describes the Grafana Loki architecture. weight: 400 aliases: - ../architecture/ @@ -10,8 +10,8 @@ aliases: # Loki architecture Grafana Loki has a microservices-based architecture and is designed to run as a horizontally scalable, distributed system. -The system has multiple components that can run separately and in parallel. -Grafana Loki's design compiles the code for all components into a single binary or Docker image. +The system has multiple components that can run separately and in parallel. The +Grafana Loki design compiles the code for all components into a single binary or Docker image. The `-target` command-line flag controls which component(s) that binary will behave as. To get started easily, run Grafana Loki in "single binary" mode with all components running simultaneously in one process, or in "simple scalable deployment" mode, which groups components into read, write, and backend parts. @@ -20,7 +20,7 @@ Grafana Loki is designed to easily redeploy a cluster under a different mode as For more information, refer to [Deployment modes]({{< relref "./deployment-modes" >}}) and [Components]({{< relref "./components" >}}). -![Loki's components](../loki_architecture_components.svg "Loki's components") +![Loki components](../loki_architecture_components.svg "Loki components") ## Storage diff --git a/docs/sources/get-started/labels/_index.md b/docs/sources/get-started/labels/_index.md index db918450bd9e1..96625b2b13ab6 100644 --- a/docs/sources/get-started/labels/_index.md +++ b/docs/sources/get-started/labels/_index.md @@ -123,7 +123,7 @@ Now instead of a regex, we could do this: Hopefully now you are starting to see the power of labels. By using a single label, you can query many streams. By combining several different labels, you can create very flexible log queries. -Labels are the index to Loki's log data. They are used to find the compressed log content, which is stored separately as chunks. Every unique combination of label and values defines a stream, and logs for a stream are batched up, compressed, and stored as chunks. +Labels are the index to Loki log data. They are used to find the compressed log content, which is stored separately as chunks. Every unique combination of label and values defines a stream, and logs for a stream are batched up, compressed, and stored as chunks. For Loki to be efficient and cost-effective, we have to use labels responsibly. The next section will explore this in more detail. diff --git a/docs/sources/get-started/overview.md b/docs/sources/get-started/overview.md index 1194398c38f0c..15fc20f330f2c 100644 --- a/docs/sources/get-started/overview.md +++ b/docs/sources/get-started/overview.md @@ -32,7 +32,7 @@ A typical Loki-based logging stack consists of 3 components: - **Scalability** - Loki is designed for scalability, and can scale from as small as running on a Raspberry Pi to ingesting petabytes a day. In its most common deployment, “simple scalable mode”, Loki decouples requests into separate read and write paths, so that you can independently scale them, which leads to flexible large-scale installations that can quickly adapt to meet your workload at any given time. -If needed, each of Loki's components can also be run as microservices designed to run natively within Kubernetes. +If needed, each of the Loki components can also be run as microservices designed to run natively within Kubernetes. - **Multi-tenancy** - Loki allows multiple tenants to share a single Loki instance. With multi-tenancy, the data and requests of each tenant is completely isolated from the others. Multi-tenancy is [configured]({{< relref "../operations/multi-tenancy" >}}) by assigning a tenant ID in the agent. @@ -44,7 +44,7 @@ Similarly, the Loki index, because it indexes only the set of labels, is signifi By leveraging object storage as the only data storage mechanism, Loki inherits the reliability and stability of the underlying object store. It also capitalizes on both the cost efficiency and operational simplicity of object storage over other storage mechanisms like locally attached solid state drives (SSD) and hard disk drives (HDD). The compressed chunks, smaller index, and use of low-cost object storage, make Loki less expensive to operate. -- **LogQL, Loki's query language** - [LogQL]({{< relref "../query" >}}) is the query language for Loki. Users who are already familiar with the Prometheus query language, [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/), will find LogQL familiar and flexible for generating queries against the logs. +- **LogQL, the Loki query language** - [LogQL]({{< relref "../query" >}}) is the query language for Loki. Users who are already familiar with the Prometheus query language, [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/), will find LogQL familiar and flexible for generating queries against the logs. The language also facilitates the generation of metrics from log data, a powerful feature that goes well beyond log aggregation. diff --git a/docs/sources/get-started/quick-start.md b/docs/sources/get-started/quick-start.md index f459e564092e1..9fa0e045da004 100644 --- a/docs/sources/get-started/quick-start.md +++ b/docs/sources/get-started/quick-start.md @@ -2,109 +2,214 @@ title: Quickstart to run Loki locally menuTitle: Loki quickstart weight: 200 -description: How to create and use a simple local Loki cluster for testing and evaluation purposes. +description: How to create and use a local Loki cluster for testing and evaluation purposes. +killercoda: + comment: | + The killercoda front matter and the HTML comments that start ' + # Quickstart to run Loki locally If you want to experiment with Loki, you can run Loki locally using the Docker Compose file that ships with Loki. It runs Loki in a [monolithic deployment](https://grafana.com/docs/loki//get-started/deployment-modes/#monolithic-mode) mode and includes a sample application to generate logs. -The Docker Compose configuration instantiates the following components, each in its own container: +The Docker Compose configuration runs the following components, each in its own container: + +- **flog**: which generates log lines. + [flog](https://github.com/mingrammer/flog) is a log generator for common log formats. + +- **Grafana Alloy**: which scrapes the log lines from flog, and pushes them to Loki through the gateway. +- **Gateway** (nginx) which receives requests and redirects them to the appropriate container based on the request's URL. +- **Loki read component**: which runs a Query Frontend and a Querier. +- **Loki write component**: which runs a Distributor and an Ingester. +- **Loki backend component**: which runs an Index Gateway, Compactor, Ruler, Bloom Compactor (experimental), and Bloom Gateway (experimental). +- **Minio**: which Loki uses to store its index and chunks. +- **Grafana**: which provides visualization of the log lines captured within Loki. + +{{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v3.png" caption="Getting started sample application" alt="Getting started sample application" >}} + + + +## Before you begin -- **flog** a sample application which generates log lines. [flog](https://github.com/mingrammer/flog) is a log generator for common log formats. -- **Grafana Alloy** which scrapes the log lines from flog, and pushes them to Loki through the gateway. -- **Gateway** (NGINX) which receives requests and redirects them to the appropriate container based on the request's URL. -- One Loki **read** component (Query Frontend, Querier). -- One Loki **write** component (Distributor, Ingester). -- One Loki **backend** component (Index Gateway, Compactor, Ruler, Bloom Compactor (Experimental), Bloom Gateway (Experimental)). -- **Minio** an S3-compatible object store which Loki uses to store its index and chunks. -- **Grafana** which provides visualization of the log lines captured within Loki. +{{< admonition type="tip" >}} +Alternatively, you can try out this example in our interactive learning environment: [Loki Quickstart Sandbox](https://killercoda.com/grafana-labs/course/loki/loki-quickstart). -{{< figure max-width="75%" src="/media/docs/loki/get-started-flog-v3.png" caption="Getting started sample application" alt="Getting started sample application">}} +It's a fully configured environment with all the dependencies already installed. + +![Interactive](https://raw.githubusercontent.com/grafana/killercoda/prod/assets/loki-ile.svg) +{{< /admonition >}} + +- Install [Docker](https://docs.docker.com/install) +- Install [Docker Compose](https://docs.docker.com/compose/install) ## Interactive Learning Environment {{< admonition type="note" >}} -The Interactive Learning Environment is currently in trial. Please provide feedback, report bugs, and raise issues in the [Grafana Killercoda Repository](https://github.com/grafana/killercoda). +The Interactive Learning Environment is in trial. + +Provide feedback, report bugs, and raise issues in the [Grafana Killercoda repository](https://github.com/grafana/killercoda). {{< /admonition >}} -Try out this demo within our interactive learning environment: [Loki Quickstart Sandbox](https://killercoda.com/grafana-labs/course/loki/loki-quickstart) +Try out this demo within our interactive learning environment: [Loki Quickstart Sandbox](https://killercoda.com/grafana-labs/course/loki/loki-quickstart) -- A free Killercoda account is required to verify you are not a bot. -- Tutorial instructions are located on the left-hand side of the screen. Click to move on to the next section. -- All commands run inside the interactive terminal. Grafana can also be accessed via the URL links provided within the sandbox. +- You must have a free Killercoda account to verify you aren't a bot. +- Tutorial instructions are located on the left-side of the screen. + Click to move on to the next section. +- All commands run inside the interactive terminal. +- You can access Grafana with the URL links provided within the sandbox. -## Installing Loki and collecting sample logs + -Prerequisites +## Install Loki and collecting sample logs -- [Docker](https://docs.docker.com/install) -- [Docker Compose](https://docs.docker.com/compose/install) + -{{% admonition type="note" %}} +{{< admonition type="note" >}} This quickstart assumes you are running Linux. -{{% /admonition %}} +{{< /admonition >}} + + **To install Loki locally, follow these steps:** -1. Create a directory called `evaluate-loki` for the demo environment. Make `evaluate-loki` your current working directory: +1. Create a directory called `evaluate-loki` for the demo environment. + Make `evaluate-loki` your current working directory: + + - ```bash - mkdir evaluate-loki - cd evaluate-loki - ``` + ```bash + mkdir evaluate-loki + cd evaluate-loki + ``` + + 1. Download `loki-config.yaml`, `alloy-local-config.yaml`, and `docker-compose.yaml`: - ```bash - wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/loki-config.yaml -O loki-config.yaml - wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/alloy-local-config.yaml -O alloy-local-config.yaml - wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/docker-compose.yaml -O docker-compose.yaml - ``` + + + ```bash + wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/loki-config.yaml -O loki-config.yaml + wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/alloy-local-config.yaml -O alloy-local-config.yaml + wget https://raw.githubusercontent.com/grafana/loki/main/examples/getting-started/docker-compose.yaml -O docker-compose.yaml + ``` + + 1. Deploy the sample Docker image. - With `evaluate-loki` as the current working directory, start the demo environment using `docker compose`: + With `evaluate-loki` as the current working directory, start the demo environment using `docker compose`: + + + + ```bash + docker compose up -d + ``` + + + + {{< docs/ignore >}} + + + ```bash + docker-compose up -d + ``` - ```bash - docker compose up -d - ``` + - You should see something similar to the following: + {{< /docs/ignore >}} - ```bash - ✔ Network evaluate-loki_loki Created 0.1s - ✔ Container evaluate-loki-minio-1 Started 0.6s - ✔ Container evaluate-loki-flog-1 Started 0.6s - ✔ Container evaluate-loki-backend-1 Started 0.8s - ✔ Container evaluate-loki-write-1 Started 0.8s - ✔ Container evaluate-loki-read-1 Started 0.8s - ✔ Container evaluate-loki-gateway-1 Started 1.1s - ✔ Container evaluate-loki-grafana-1 Started 1.4s - ✔ Container evaluate-loki-alloy-1 Started 1.4s - ``` + At the end of the command, you should see something similar to the following: + + + + ```console + ✔ Network evaluate-loki_loki Created 0.1s + ✔ Container evaluate-loki-minio-1 Started 0.6s + ✔ Container evaluate-loki-flog-1 Started 0.6s + ✔ Container evaluate-loki-backend-1 Started 0.8s + ✔ Container evaluate-loki-write-1 Started 0.8s + ✔ Container evaluate-loki-read-1 Started 0.8s + ✔ Container evaluate-loki-gateway-1 Started 1.1s + ✔ Container evaluate-loki-grafana-1 Started 1.4s + ✔ Container evaluate-loki-alloy-1 Started 1.4s + ``` + + + + {{< docs/ignore >}} + + ```console + Creating evaluate-loki_flog_1 ... done + Creating evaluate-loki_minio_1 ... done + Creating evaluate-loki_read_1 ... done + Creating evaluate-loki_write_1 ... done + Creating evaluate-loki_gateway_1 ... done + Creating evaluate-loki_alloy_1 ... done + Creating evaluate-loki_grafana_1 ... done + Creating evaluate-loki_backend_1 ... done + ``` + + {{< /docs/ignore >}} 1. (Optional) Verify that the Loki cluster is up and running. - - The read component returns `ready` when you point a web browser at [http://localhost:3101/ready](http://localhost:3101/ready). The message `Query Frontend not ready: not ready: number of schedulers this worker is connected to is 0` will show prior to the read component being ready. - - The write component returns `ready` when you point a web browser at [http://localhost:3102/ready](http://localhost:3102/ready). The message `Ingester not ready: waiting for 15s after being ready` will show prior to the write component being ready. - + + - The read component returns `ready` when you browse to [http://localhost:3101/ready](http://localhost:3101/ready). + The message `Query Frontend not ready: not ready: number of schedulers this worker is connected to is 0` shows until the read component is ready. + - The write component returns `ready` when you browse to [http://localhost:3102/ready](http://localhost:3102/ready). + The message `Ingester not ready: waiting for 15s after being ready` shows until the write component is ready. + 1. (Optional) Verify that Grafana Alloy is running. - - Grafana Alloy's UI can be accessed at [http://localhost:12345](http://localhost:12345). + - You can access the Grafana Alloy UI at [http://localhost:12345](http://localhost:12345). + + + + + +## View your logs in Grafana -## Viewing your logs in Grafana +After you have collected logs, you will want to view them. +You can view your logs using the command line interface, [LogCLI](/docs/loki//query/logcli/), but the easiest way to view your logs is with Grafana. -Once you have collected logs, you will want to view them. You can view your logs using the command line interface, [LogCLI](/docs/loki//query/logcli/), but the easiest way to view your logs is with Grafana. +1. Use Grafana to query the Loki data source. -1. Use Grafana to query the Loki data source. + The test environment includes [Grafana](https://grafana.com/docs/grafana/latest/), which you can use to query and observe the sample logs generated by the flog application. - The test environment includes [Grafana](https://grafana.com/docs/grafana/latest/), which you can use to query and observe the sample logs generated by the flog application. You can access the Grafana cluster by navigating to [http://localhost:3000](http://localhost:3000). The Grafana instance provided with this demo has a Loki [datasource](https://grafana.com/docs/grafana/latest/datasources/loki/) already configured. + You can access the Grafana cluster by browsing to [http://localhost:3000](http://localhost:3000). - {{< figure src="/media/docs/loki/grafana-query-builder-v2.png" caption="Grafana Explore" alt="Grafana Explore">}} + The Grafana instance in this demonstration has a Loki [data source](https://grafana.com/docs/grafana/latest/datasources/loki/) already configured. -1. From the Grafana main menu, click the **Explore** icon (1) to launch the Explore tab. To learn more about Explore, refer the [Explore](https://grafana.com/docs/grafana/latest/explore/) documentation. + {{< figure src="/media/docs/loki/grafana-query-builder-v2.png" caption="Grafana Explore" alt="Grafana Explore" >}} -1. From the menu in the dashboard header, select the Loki data source (2). This displays the Loki query editor. In the query editor you use the Loki query language, [LogQL](https://grafana.com/docs/loki//query/), to query your logs. - To learn more about the query editor, refer to the [query editor documentation](https://grafana.com/docs/grafana/latest/datasources/loki/query-editor/). +1. From the Grafana main menu, click the **Explore** icon (1) to open the Explore tab. + + To learn more about Explore, refer to the [Explore](https://grafana.com/docs/grafana/latest/explore/) documentation. + +1. From the menu in the dashboard header, select the Loki data source (2). + + This displays the Loki query editor. + + In the query editor you use the Loki query language, [LogQL](https://grafana.com/docs/loki//query/), to query your logs. + To learn more about the query editor, refer to the [query editor documentation](https://grafana.com/docs/grafana/latest/datasources/loki/query-editor/). 1. The Loki query editor has two modes (3): @@ -115,47 +220,61 @@ Once you have collected logs, you will want to view them. You can view your log 1. Click **Code** (3) to work in Code mode in the query editor. - Here are some basic sample queries to get you started using LogQL. Note that these queries assume that you followed the instructions to create a directory called `evaluate-loki`. If you installed in a different directory, you’ll need to modify these queries to match your installation directory. After copying any of these queries into the query editor, click **Run Query** (4) to execute the query. + Here are some sample queries to get you started using LogQL. + These queries assume that you followed the instructions to create a directory called `evaluate-loki`. + + If you installed in a different directory, you’ll need to modify these queries to match your installation directory. - 1. View all the log lines which have the container label "flog": + After copying any of these queries into the query editor, click **Run Query** (4) to execute the query. - ```bash - {container="evaluate-loki-flog-1"} - ``` + 1. View all the log lines which have the container label `evaluate-loki-flog-1`: - In Loki, this is called a log stream. Loki uses [labels](https://grafana.com/docs/loki//get-started/labels/) as metadata to describe log streams. Loki queries always start with a label selector. In the query above, the label selector is `container`. + ```bash + {container="evaluate-loki-flog-1"} + ``` - 1. To view all the log lines which have the container label "grafana": + In Loki, this is a log stream. - ```bash - {container="evaluate-loki-grafana-1"} - ``` + Loki uses [labels](https://grafana.com/docs/loki//get-started/labels/) as metadata to describe log streams. - 1. Find all the log lines in the container=flog stream that contain the string "status": + Loki queries always start with a label selector. + In the previous query, the label selector is `{container="evaluate-loki-flog-1"}`. - ```bash - {container="evaluate-loki-flog-1"} |= `status` - ``` + 1. To view all the log lines which have the container label `evaluate-loki-grafana-1`: - 1. Find all the log lines in the container=flog stream where the JSON field "status" is "404": + ```bash + {container="evaluate-loki-grafana-1"} + ``` - ```bash - {container="evaluate-loki-flog-1"} | json | status=`404` - ``` + 1. Find all the log lines in the `{container="evaluate-loki-flog-1"}` stream that contain the string `status`: - 1. Calculate the number of logs per second where the JSON field "status" is "404": + ```bash + {container="evaluate-loki-flog-1"} |= `status` + ``` - ```bash - sum by(container) (rate({container="evaluate-loki-flog-1"} | json | status=`404` [$__auto])) - ``` + 1. Find all the log lines in the `{container="evaluate-loki-flog-1"}` stream where the JSON field `status` has the value `404`: - The final query above is a metric query which returns a time series. This will trigger Grafana to draw a graph of the results. You can change the type of graph for a different view of the data. Click **Bars** to view a bar graph of the data. + ```bash + {container="evaluate-loki-flog-1"} | json | status=`404` + ``` -1. Click the **Builder** tab (3) to return to Builder mode in the query editor. - 1. In Builder view, click **Kick start your query**(5). - 1. Expand the **Log query starters** section. - 1. Select the first choice, **Parse log lines with logfmt parser**, by clicking **Use this query**. - 1. On the Explore tab, click **Label browser**, in the dialog select a container and click **Show logs**. + 1. Calculate the number of logs per second where the JSON field `status` has the value `404`: + + ```bash + sum by(container) (rate({container="evaluate-loki-flog-1"} | json | status=`404` [$__auto])) + ``` + + The final query is a metric query which returns a time series. + This makes Grafana draw a graph of the results. + + You can change the type of graph for a different view of the data. + Click **Bars** to view a bar graph of the data. + +1. Click the **Builder** tab (3) to return to builder mode in the query editor. + 1. In builder mode, click **Kick start your query** (5). + 1. Expand the **Log query starters** section. + 1. Select the first choice, **Parse log lines with logfmt parser**, by clicking **Use this query**. + 1. On the Explore tab, click **Label browser**, in the dialog select a container and click **Show logs**. For a thorough introduction to LogQL, refer to the [LogQL reference](https://grafana.com/docs/loki//query/). @@ -166,7 +285,7 @@ Here are some more sample queries that you can run using the Flog sample data. To see all the log lines that flog has generated, enter the LogQL query: ```bash -{container="evaluate-loki-flog-1"}|= `` +{container="evaluate-loki-flog-1"} ``` The flog app generates log lines for simulated HTTP requests. @@ -189,7 +308,7 @@ To see every log line with a 401 status (unauthorized error), enter the LogQL qu {container="evaluate-loki-flog-1"} | json | status="401" ``` -To see every log line that does not contain the value 401: +To see every log line that doesn't contain the text `401`: ```bash {container="evaluate-loki-flog-1"} != "401" @@ -197,8 +316,12 @@ To see every log line that does not contain the value 401: For more examples, refer to the [query documentation](https://grafana.com/docs/loki//query/query_examples/). + + ## Complete metrics, logs, traces, and profiling example -If you would like to use a demo that includes Mimir, Loki, Tempo, and Grafana, you can use [Introduction to Metrics, Logs, Traces, and Profiling in Grafana](https://github.com/grafana/intro-to-mlt). `Intro-to-mltp` provides a self-contained environment for learning about Mimir, Loki, Tempo, and Grafana. +If you would like to run a demonstration environment that includes Mimir, Loki, Tempo, and Grafana, you can use [Introduction to Metrics, Logs, Traces, and Profiling in Grafana](https://github.com/grafana/intro-to-mlt). +It's a self-contained environment for learning about Mimir, Loki, Tempo, and Grafana. -The project includes detailed explanations of each component and annotated configurations for a single-instance deployment. Data from `intro-to-mltp` can also be pushed to Grafana Cloud. +The project includes detailed explanations of each component and annotated configurations for a single-instance deployment. +You can also push the data from the environment to [Grafana Cloud](https://grafana.com/cloud/). diff --git a/docs/sources/operations/authentication.md b/docs/sources/operations/authentication.md index 96081dbab52e7..11949a1a9811a 100644 --- a/docs/sources/operations/authentication.md +++ b/docs/sources/operations/authentication.md @@ -1,7 +1,7 @@ --- title: Authentication menuTitle: -description: Describes Loki's authentication. +description: Describes Loki authentication. weight: --- # Authentication diff --git a/docs/sources/operations/meta-monitoring/_index.md b/docs/sources/operations/meta-monitoring/_index.md new file mode 100644 index 0000000000000..7b90955ef2ad4 --- /dev/null +++ b/docs/sources/operations/meta-monitoring/_index.md @@ -0,0 +1,103 @@ +--- +title: Monitor Loki +description: Describes the various options for monitoring your Loki environment, and the metrics available. +aliases: + - ../operations/observability +--- + +# Monitor Loki + +As part of your Loki implementation, you will also want to monitor your Loki cluster. + +As a best practice, you should collect data about Loki in a separate instance of Loki, for example, send your Loki data to a [Grafana Cloud account](https://grafana.com/products/cloud/). This will let you troubleshoot a broken Loki cluster from a working one. + +Loki exposes the following observability data about itself: + +- **Metrics**: Loki provides a `/metrics` endpoint that exports information about Loki in Prometheus format. These metrics provide aggregated metrics of the health of your Loki cluster, allowing you to observe query response times, etc etc. +- **Logs**: Loki emits a detailed log line `metrics.go` for every query, which shows query duration, number of lines returned, query throughput, the specific LogQL that was executed, chunks searched, and much more. You can use these log lines to improve and optimize your query performance. + +You can also scrape the Loki logs and metrics and push them to separate instances of Loki and Mimir to provide information about the health of your Loki system (a process known as "meta-monitoring"). + +The Loki [mixin](https://github.com/grafana/loki/blob/main/production/loki-mixin) is an opinionated set of dashboards, alerts and recording rules to monitor your Loki cluster. The mixin provides a comprehensive package for monitoring Loki in production. You can install the mixin into a Grafana instance. + +- To install meta-monitoring using the Loki Helm Chart and Grafana Cloud, follow [these directions](https://grafana.com/docs/loki//setup/install/helm/monitor-and-alert/with-grafana-cloud/). + +- To install meta-monitoring using the Loki Helm Chart and a local Loki stack, follow [these directions](https://grafana.com/docs/loki//setup/install/helm/monitor-and-alert/with-local-monitoring/). + +- To install the Loki mixin, follow [these directions]({{< relref "./mixins" >}}). + +You should also plan separately for infrastructure-level monitoring, to monitor the capacity or throughput of your storage provider, for example, or your networking layer. + +- [MinIO](https://min.io/docs/minio/linux/operations/monitoring/collect-minio-metrics-using-prometheus.html) +- [Kubernetes](https://grafana.com/docs/grafana-cloud/monitor-infrastructure/kubernetes-monitoring/) + +## Loki Metrics + +As Loki is a [distributed system](https://grafana.com/docs/loki//get-started/components/), each component exports its own metrics. The `/metrics` endpoint exposes hundreds of different metrics. You can find a sampling of the metrics exposed by Loki and their descriptions, in the sections below. + +You can find a complete list of the exposed metrics by checking the `/metrics` endpoint. + +`http://:/metrics` + +For example: + +[http://localhost:3100/metrics](http://localhost:3100/metrics) + +Both Grafana Loki and Promtail expose a `/metrics` endpoint that expose Prometheus metrics (the default port is 3100 for Loki and 80 for Promtail). You will need a local Prometheus and add Loki and Promtail as targets. See [configuring Prometheus](https://prometheus.io/docs/prometheus/latest/configuration/configuration) for more information. + +All components of Loki expose the following metrics: + +| Metric Name | Metric Type | Description | +| ---------------------------------- | ----------- | ----------------------------------------------------------------------- | +| `loki_internal_log_messages_total` | Counter | Total number of log messages created by Loki itself. | +| `loki_request_duration_seconds` | Histogram | Number of received HTTP requests. | + +Note that most of the metrics are counters and should continuously increase during normal operations. + +1. Your app emits a log line to a file that is tracked by Promtail. +1. Promtail reads the new line and increases its counters. +1. Promtail forwards the log line to a Loki distributor, where the received + counters should increase. +1. The Loki distributor forwards the log line to a Loki ingester, where the + request duration counter should increase. + +If Promtail uses any pipelines with metrics stages, those metrics will also be +exposed by Promtail at its `/metrics` endpoint. See Promtail's documentation on +[Pipelines](https://grafana.com/docs/loki//send-data/promtail/pipelines/) for more information. + +### Metrics cardinality + +Some of the Loki observability metrics are emitted per tracked file (active), with the file path included in labels. This increases the quantity of label values across the environment, thereby increasing cardinality. Best practices with Prometheus labels discourage increasing cardinality in this way. Review your emitted metrics before scraping with Prometheus, and configure the scraping to avoid this issue. + +## Example Loki log line: metrics.go + +Loki emits a "metrics.go" log line from the Querier, Query frontend and Ruler components, which lets you inspect query and recording rule performance. This is an example of a detailed log line "metrics.go" for a query. + +Example log + +`level=info ts=2024-03-11T13:44:10.322919331Z caller=metrics.go:143 component=frontend org_id=mycompany latency=fast query="sum(count_over_time({kind=\"auditing\"} | json | user_userId =`` [1m]))" query_type=metric range_type=range length=10m0s start_delta=10m10.322900424s end_delta=10.322900663s step=1s duration=47.61044ms status=200 limit=100 returned_lines=0 throughput=9.8MB total_bytes=467kB total_entries=1 queue_time=0s subqueries=2 cache_chunk_req=1 cache_chunk_hit=1 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=14394 cache_index_req=19 cache_index_hit=19 cache_result_req=1 cache_result_hit=1` + +You can use the query-frontend `metrics.go` lines to understand a query’s overall performance. The “metrics.go” line output by the Queriers contains the same information as the Query frontend but is often more helpful in understanding and troubleshooting query performance. This is largely because it can tell you how the querier spent its time executing the subquery. Here are the most useful stats: + +- **total_bytes**: how many total bytes the query processed +- **duration**: how long the query took to execute +- **throughput**: total_bytes/duration +- **total_lines**: how many total lines the query processed +- **length**: how much time the query was executed over +- **post_filter_lines**: how many lines matched the filters in the query +- **cache_chunk_req**: total number of chunks fetched for the query (the cache will be asked for every chunk so this is equivalent to the total chunks requested) +- **splits**: how many pieces the query was split into based on time and split_queries_by_interval +- **shards**: how many shards the query was split into + +For more information, refer to the blog post [The concise guide to Loki: How to get the most out of your query performance](https://grafana.com/blog/2023/12/28/the-concise-guide-to-loki-how-to-get-the-most-out-of-your-query-performance/). + +### Configure Logging Levels + +To change the configuration for Loki logging levels, update log_level configuration parameter in your `config.yaml` file. + +```yaml +# Only log messages with the given severity or above. Valid levels: [debug, +# info, warn, error] +# CLI flag: -log.level +[log_level: | default = "info"] +``` diff --git a/docs/sources/operations/meta-monitoring/mixins.md b/docs/sources/operations/meta-monitoring/mixins.md new file mode 100644 index 0000000000000..a4a819c4e3d28 --- /dev/null +++ b/docs/sources/operations/meta-monitoring/mixins.md @@ -0,0 +1,189 @@ +--- +title: Install Loki mixins +menuTitle: Install mixins +description: Describes the Loki mixins, how to configure and install the dashboards, alerts, and recording rules. +weight: 100 +--- + +# Install Loki mixins + +Loki is instrumented to expose metrics about itself via the `/metrics` endpoint, designed to be scraped by Prometheus. Each Loki release includes a mixin. The Loki mixin provides a set of Grafana dashboards, Prometheus recording rules and alerts for monitoring Loki. + +To set up monitoring using the mixin, you need to: + +- Deploy an instance of Prometheus (or a Prometheus-compatible time series database, like [Mimir](https://grafana.com/docs/mimir/latest/)) which can store Loki metrics. +- Deploy an agent, such as Grafana Alloy, or Grafana Agent, to scrape Loki metrics. +- Set up Grafana to visualize Loki metrics, by installing the dashboards. +- Install the recording rules and alerts into Prometheus using `mimirtool`. + +This procedure assumes that you have set up Loki using the Helm chart. + +{{< admonition type="note" >}} +Be sure to update the commands and configuration to match your own deployment. +{{< /admonition >}} + +## Before you begin + +To make full use of the Loki mixin, you’ll need the following running in your environment: + +- Loki instance - A Loki instance which you want to monitor. +- Grafana - For visualizing logs and metrics ([install on Kubernetes](https://grafana.com/docs/grafana/latest/setup-grafana/installation/kubernetes/#deploy-grafana-oss-on-kubernetes)). +- Prometheus or Mimir - An instance of Prometheus or Mimir which will store metrics from Loki. + +To scrape metrics from Loki, you can use Grafana Alloy or the OpenTelemetry Collector. This procedure provides examples only for Grafana Alloy. + +If you have installed Loki using a Helm Chart, this documentation assumes that the Loki and Grafana instances are located on the same Kubernetes cluster. + +## Configure Alloy to scrape Loki metrics + +Loki exposes Prometheus metrics from all of its components to allow meta-monitoring. To retrieve these metrics, you need to configure a suitable scraper. Grafana Alloy can collect metrics and act as a Prometheus scraper. To use this capability, you need to configure Alloy to scrape from all of the components. + +{{< admonition type="tip" >}} +If you're running on Kubernetes, you can use the Kubernetes Monitoring Helm chart. +{{< /admonition >}} + +To scrape metrics from Loki, follow these steps: + +Install Grafana Alloy using the provided instructions for your platform. + +- [Standalone](https://grafana.com/docs/alloy/latest/get-started/install/binary/) +- [Kubernetes](https://grafana.com/docs/alloy/latest/get-started/install/kubernetes/) +- [Docker](https://grafana.com/docs/alloy/latest/get-started/install/docker/) + +Add a configuration block to scrape metrics from your Loki component instances and forward to a Prometheus or Mimir instance. + +- On Kubernetes, you can use the Alloy `discovery.kubernetes` component to discover Loki Pods to scrape metrics from. +- On non-Kubernetes deployments, you may use `prometheus.scrape` and an explicit list of targets to discover Loki instances to scrape. + +For an example, see [Collect and forward Prometheus metrics](https://grafana.com/docs/alloy/latest/tasks/collect-prometheus-metrics/). + +## Configure Grafana + +In your Grafana instance, you'll need to [create a Prometheus data source](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure-prometheus-data-source/) to visualize the metrics scraped from your Loki cluster. + +## Install Loki dashboards in Grafana + +After Loki metrics are scraped by Grafana Alloy and stored in a Prometheus compatible time-series database, you can monitor Loki’s operation using the Loki mixin. + +Each Loki release includes a mixin that includes: + +- Relevant dashboards for overseeing the health of Loki as a whole, as well as its individual Loki components +- [Recording rules](https://grafana.com/docs/loki/latest/alert/#recording-rules) that compute metrics that are used in the dashboards +- Alerts that trigger when Loki generates metrics that are outside of normal parameters + +To install the mixins in Grafana and Mimir, the general steps are as follows: + +1. Download the mixin dashboards from the Loki repository. + +1. Import the dashboards in your Grafana instance. + +1. Upload `alerts.yaml` and `rules.yaml` files to Prometheus or Mimir with `mimirtool`. + +### Download the `loki-mixin` dashboards + +1. First, clone the Loki repository from Github: + + ```bash + git clone https://github.com/grafana/loki + cd loki + ``` + +1. Once you have a local copy of the repository, navigate to the `production/loki-mixin-compiled-ssd` directory. + + ```bash + cd production/loki-mixin-compiled-ssd + ``` + + OR, if you're deploying Loki in microservices mode: + + ```bash + cd production/loki-mixin-compiled + ``` + +This directory contains a compiled version of the alert and recording rules, as well as the dashboards. + +{{< admonition type="note" >}} +If you want to change any of the mixins, make your updates in the `production/loki-mixin` directory. +Use the instructions in the [README](https://github.com/grafana/loki/tree/main/production/loki-mixin) in that directory to regenerate the files. +{{< /admonition >}} + +### Import the dashboards to Grafana + +The `dashboards` directory includes the monitoring dashboards that can be installed into your Grafana instance. +Refer to [Import a dashboard](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/) in the Grafana documentation. + +{{< admonition type="tip" >}} +Install all dashboards. +You can only import one dashboard at a time. +Create a new folder in the Dashboards area, for example “Loki Monitoring”, as an easy location to save the imported dashboards. +{{< /admonition >}} + +To create a folder: + +1. Open your Grafana instance and select **Dashboards**. +1. Click the **New** button. +1. Select **New folder** from the **New** menu. +1. Name your folder, for example, “Loki Monitoring”. +1. Click **Create**. + +To import a dashboard: + +1. Open your Grafana instance and select **Dashboards**. +1. Click the **New** button. +1. Select **Import** from the **New** menu. +1. On the **Import dashboard** screen, select **Upload dashboard JSON file.** +1. Browse to `production/loki-mixin-compiled-ssd/dashboards` and select the dashboard to import. Or, drag the dashboard file, for example, `loki-operational.json`, onto the **Upload** area of the **Import dashboard** screen. +1. Select a folder in the **Folder** menu where you want to save the imported dashboard. For example, select "Loki Monitoring" created in the earlier steps. +1. Click **Import**. + +The imported files are listed in the Loki Monitoring dashboard folder. + +To view the dashboards in Grafana: + +1. Select **Dashboards** in your Grafana instance. +1. Select **Loki Monitoring**, or the folder where you uploaded the imported dashboards. +1. Select any file in the folder to view the dashboard. + +### Add alerts and recording rules to Prometheus or Mimir + +The rules and alerts need to be installed into a Prometheus instance, Mimir or a Grafana Enterprise Metrics cluster. + +You can find the YAML files for alerts and rules in the following directories in the Loki repo: + +For SSD mode: +`production/loki-mixin-compiled-ssd/alerts.yaml` +`production/loki-mixin-compiled-ssd/rules.yaml` + +For microservices mode: +`production/loki-mixin-compiled/alerts.yaml` +`production/loki-mixin-compiled/rules.yaml` + +You use `mimirtool` to load the mixin alerts and rules definitions into a Prometheus instance, Mimir or a Grafana Enterprise Metrics cluster. + +1. Download [mimirtool](https://github.com/grafana/mimir/releases). + +1. Using the details of a Prometheus instance or Mimir cluster, run the following command to load the recording rules: + + ```bash + mimirtool rules load --address=http://prometheus:9090 rules.yaml + ``` + + Or, for example if your Mimir cluster requires an API key, as is the case with Grafana Enterprise Metrics: + + ```bash + mimirtool rules load --id= --address=http://: --key="" rules.yaml + ``` + +1. To load alerts: + + ```bash + mimirtool alertmanager load --address=http://prometheus:9090 alerts.yaml + ``` + + or + + ```bash + mimirtool alertmanager load --id= --address=http://: --key="" alerts.yaml + ``` + +Refer to the [mimirtool](https://grafana.com/docs/mimir/latest/manage/tools/mimirtool/) documentation for more information. diff --git a/docs/sources/operations/observability.md b/docs/sources/operations/observability.md deleted file mode 100644 index 8f617bcf869dc..0000000000000 --- a/docs/sources/operations/observability.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Observability -menuTitle: -description: Observing Grafana Loki -weight: ---- -# Observability - -Both Grafana Loki and Promtail expose a `/metrics` endpoint that expose Prometheus -metrics (the default port is 3100 for Loki and 80 for Promtail). You will need -a local Prometheus and add Loki and Promtail as targets. See [configuring -Prometheus](https://prometheus.io/docs/prometheus/latest/configuration/configuration) -for more information. - -All components of Loki expose the following metrics: - -| Metric Name | Metric Type | Description | -| ---------------------------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------- | -| `loki_log_messages_total` | Counter | DEPRECATED. Use internal_log_messages_total for the same functionality. Total number of log messages created by Loki itself. | -| `loki_internal_log_messages_total` | Counter | Total number of log messages created by Loki itself. | -| `loki_request_duration_seconds` | Histogram | Number of received HTTP requests. | - -The Loki Distributors expose the following metrics: - -| Metric Name | Metric Type | Description | -| ------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `loki_distributor_ingester_appends_total` | Counter | The total number of batch appends sent to ingesters. | -| `loki_distributor_ingester_append_failures_total` | Counter | The total number of failed batch appends sent to ingesters. | -| `loki_distributor_bytes_received_total` | Counter | The total number of uncompressed bytes received per both tenant and retention hours. | -| `loki_distributor_lines_received_total` | Counter | The total number of log _entries_ received per tenant (not necessarily of _lines_, as an entry can have more than one line of text). | - -The Loki Ingesters expose the following metrics: - -| Metric Name | Metric Type | Description | -| -------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------- | -| `loki_ingester_flush_queue_length` | Gauge | The total number of series pending in the flush queue. | -| `loki_chunk_store_index_entries_per_chunk` | Histogram | Number of index entries written to storage per chunk. | -| `loki_ingester_memory_chunks` | Gauge | The total number of chunks in memory. | -| `loki_ingester_memory_streams` | Gauge | The total number of streams in memory. | -| `loki_ingester_chunk_age_seconds` | Histogram | Distribution of chunk ages when flushed. | -| `loki_ingester_chunk_encode_time_seconds` | Histogram | Distribution of chunk encode times. | -| `loki_ingester_chunk_entries` | Histogram | Distribution of lines per-chunk when flushed. | -| `loki_ingester_chunk_size_bytes` | Histogram | Distribution of chunk sizes when flushed. | -| `loki_ingester_chunk_utilization` | Histogram | Distribution of chunk utilization (filled uncompressed bytes vs maximum uncompressed bytes) when flushed. | -| `loki_ingester_chunk_compression_ratio` | Histogram | Distribution of chunk compression ratio when flushed. | -| `loki_ingester_chunk_stored_bytes_total` | Counter | Total bytes stored in chunks per tenant. | -| `loki_ingester_chunks_created_total` | Counter | The total number of chunks created in the ingester. | -| `loki_ingester_chunks_stored_total` | Counter | Total stored chunks per tenant. | -| `loki_ingester_received_chunks` | Counter | The total number of chunks sent by this ingester whilst joining during the handoff process. | -| `loki_ingester_samples_per_chunk` | Histogram | The number of samples in a chunk. | -| `loki_ingester_sent_chunks` | Counter | The total number of chunks sent by this ingester whilst leaving during the handoff process. | -| `loki_ingester_streams_created_total` | Counter | The total number of streams created per tenant. | -| `loki_ingester_streams_removed_total` | Counter | The total number of streams removed per tenant. | - -The Loki compactor exposes the following metrics: - -| Metric Name | Metric Type | Description | -| ------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------- | -| `loki_compactor_delete_requests_processed_total` | Counter | Number of delete requests processed per user. | -| `loki_compactor_delete_requests_chunks_selected_total` | Counter | Number of chunks selected while building delete plans per user. | -| `loki_compactor_delete_processing_fails_total` | Counter | Number of times the delete phase of compaction has failed. | -| `loki_compactor_load_pending_requests_attempts_total` | Counter | Number of attempts that were made to load pending requests with status. | -| `loki_compactor_oldest_pending_delete_request_age_seconds` | Gauge | Age of oldest pending delete request in seconds since they are over their cancellation period. | -| `loki_compactor_pending_delete_requests_count` | Gauge | Count of delete requests which are over their cancellation period and have not finished processing yet. | -| `loki_compactor_deleted_lines` | Counter | Number of deleted lines per user. | - -Promtail exposes these metrics: - -| Metric Name | Metric Type | Description | -| ----------------------------------------- | ----------- | ------------------------------------------------------------------------------------------ | -| `promtail_read_bytes_total` | Gauge | Number of bytes read. | -| `promtail_read_lines_total` | Counter | Number of lines read. | -| `promtail_dropped_bytes_total` | Counter | Number of bytes dropped because failed to be sent to the ingester after all retries. | -| `promtail_dropped_entries_total` | Counter | Number of log entries dropped because failed to be sent to the ingester after all retries. | -| `promtail_encoded_bytes_total` | Counter | Number of bytes encoded and ready to send. | -| `promtail_file_bytes_total` | Gauge | Number of bytes read from files. | -| `promtail_files_active_total` | Gauge | Number of active files. | -| `promtail_request_duration_seconds` | Histogram | Number of send requests. | -| `promtail_sent_bytes_total` | Counter | Number of bytes sent. | -| `promtail_sent_entries_total` | Counter | Number of log entries sent to the ingester. | -| `promtail_targets_active_total` | Gauge | Number of total active targets. | -| `promtail_targets_failed_total` | Counter | Number of total failed targets. | - -Most of these metrics are counters and should continuously increase during normal operations: - -1. Your app emits a log line to a file that is tracked by Promtail. -2. Promtail reads the new line and increases its counters. -3. Promtail forwards the log line to a Loki distributor, where the received - counters should increase. -4. The Loki distributor forwards the log line to a Loki ingester, where the - request duration counter should increase. - -If Promtail uses any pipelines with metrics stages, those metrics will also be -exposed by Promtail at its `/metrics` endpoint. See Promtail's documentation on -[Pipelines]({{< relref "../send-data/promtail/pipelines" >}}) for more information. - -An example Grafana dashboard was built by the community and is available as -dashboard [10004](/dashboards/10004). - -## Metrics cardinality - -Some of the Loki observability metrics are emitted per tracked file (active), with the file path included in labels. -This increases the quantity of label values across the environment, thereby increasing cardinality. Best practices with Prometheus [labels](https://prometheus.io/docs/practices/naming/#labels) discourage increasing cardinality in this way. -Review your emitted metrics before scraping with Prometheus, and configure the scraping to avoid this issue. - - -## Mixins - -The Loki repository has a [mixin](https://github.com/grafana/loki/blob/main/production/loki-mixin) that includes a -set of dashboards, recording rules, and alerts. Together, the mixin gives you a -comprehensive package for monitoring Loki in production. - -For more information about mixins, take a look at the docs for the -[monitoring-mixins project](https://github.com/monitoring-mixins/docs). diff --git a/docs/sources/operations/query-acceleration-blooms.md b/docs/sources/operations/query-acceleration-blooms.md index 038a625492b26..6122fa2c709bb 100644 --- a/docs/sources/operations/query-acceleration-blooms.md +++ b/docs/sources/operations/query-acceleration-blooms.md @@ -34,6 +34,14 @@ The underlying blooms are built by the new [Bloom Compactor](#bloom-compactor) c and served by the new [Bloom Gateway](#bloom-gateway) component. ## Enable Query Acceleration with Blooms +{{< admonition type="warning" >}} +Building and querying bloom filters are by design not supported in single binary deployment. +It can be used with Single Scalable deployment (SSD), but it is recommended to +run bloom components only in fully distributed microservice mode. +The reason is that bloom filters also come with a relatively high cost for both building +and querying the bloom filters that only pays off at large scale deployments. +{{< /admonition >}} + To start building and using blooms you need to: - Deploy the [Bloom Compactor](#bloom-compactor) component and enable the component in the [Bloom Compactor config][compactor-cfg]. - Deploy the [Bloom Gateway](#bloom-gateway) component (as a [microservice][microservices] or via the [SSD][ssd] Backend target) and enable the component in the [Bloom Gateway config][gateway-cfg]. diff --git a/docs/sources/operations/query-fairness/_index.md b/docs/sources/operations/query-fairness/_index.md index 44b3c15f8f9ad..79c569d5de723 100644 --- a/docs/sources/operations/query-fairness/_index.md +++ b/docs/sources/operations/query-fairness/_index.md @@ -95,7 +95,7 @@ curl -s http://localhost:3100/loki/api/v1/query_range?xxx \ ``` There is a limit to how deep a path and thus the queue tree can be. This is -controlled by Loki's `-query-scheduler.max-queue-hierarchy-levels` CLI argument +controlled by the Loki `-query-scheduler.max-queue-hierarchy-levels` CLI argument or its respective YAML configuration block: ```yaml diff --git a/docs/sources/operations/recording-rules.md b/docs/sources/operations/recording-rules.md index 2254510daf7ee..fd5b8ae6cd5b5 100644 --- a/docs/sources/operations/recording-rules.md +++ b/docs/sources/operations/recording-rules.md @@ -11,7 +11,7 @@ Recording rules are evaluated by the `ruler` component. Each `ruler` acts as its executes queries against the store without using the `query-frontend` or `querier` components. It will respect all query [limits](https://grafana.com/docs/loki//configure/#limits_config) put in place for the `querier`. -Loki's implementation of recording rules largely reuses Prometheus' code. +The Loki implementation of recording rules largely reuses Prometheus' code. Samples generated by recording rules are sent to Prometheus using Prometheus' **remote-write** feature. diff --git a/docs/sources/operations/request-validation-rate-limits.md b/docs/sources/operations/request-validation-rate-limits.md index c5472beac3757..6d67b3d26d2c0 100644 --- a/docs/sources/operations/request-validation-rate-limits.md +++ b/docs/sources/operations/request-validation-rate-limits.md @@ -129,7 +129,7 @@ This validation error is returned when a stream is submitted without any labels. The `too_far_behind` and `out_of_order` reasons are identical. Loki clusters with `unordered_writes=true` (the default value as of Loki v2.4) use `reason=too_far_behind`. Loki clusters with `unordered_writes=false` use `reason=out_of_order`. -This validation error is returned when a stream is submitted out of order. More details can be found [here](/docs/loki//configuration/#accept-out-of-order-writes) about Loki's ordering constraints. +This validation error is returned when a stream is submitted out of order. More details can be found [here](/docs/loki//configuration/#accept-out-of-order-writes) about the Loki ordering constraints. The `unordered_writes` config value can be modified globally in the [`limits_config`](/docs/loki//configuration/#limits_config) block, or on a per-tenant basis in the [runtime overrides](/docs/loki//configuration/#runtime-configuration-file) file, whereas `max_chunk_age` is a global configuration. diff --git a/docs/sources/operations/storage/_index.md b/docs/sources/operations/storage/_index.md index b0cea23bd43d7..74fa7620d9085 100644 --- a/docs/sources/operations/storage/_index.md +++ b/docs/sources/operations/storage/_index.md @@ -1,7 +1,7 @@ --- title: Manage storage menuTitle: Storage -description: Describes Loki's storage needs and supported stores. +description: Describes the Loki storage needs and supported stores. --- # Manage storage @@ -17,7 +17,7 @@ they are compressed as **chunks** and saved in the chunks store. See [chunk format](#chunk-format) for how chunks are stored internally. The **index** stores each stream's label set and links them to the individual -chunks. Refer to Loki's [configuration](https://grafana.com/docs/loki//configure/) for +chunks. Refer to the Loki [configuration](https://grafana.com/docs/loki//configure/) for details on how to configure the storage and the index. For more information: diff --git a/docs/sources/operations/storage/legacy-storage.md b/docs/sources/operations/storage/legacy-storage.md index 66a3f76075f13..5ec0859833655 100644 --- a/docs/sources/operations/storage/legacy-storage.md +++ b/docs/sources/operations/storage/legacy-storage.md @@ -12,7 +12,7 @@ The usage of legacy storage for new installations is highly discouraged and docu purposes in case of upgrade to a single store. {{% /admonition %}} -The **chunk store** is Loki's long-term data store, designed to support +The **chunk store** is the Loki long-term data store, designed to support interactive querying and sustained writing without the need for background maintenance tasks. It consists of: diff --git a/docs/sources/operations/storage/wal.md b/docs/sources/operations/storage/wal.md index 2bf9010c948bc..be3761eff02f2 100644 --- a/docs/sources/operations/storage/wal.md +++ b/docs/sources/operations/storage/wal.md @@ -32,7 +32,7 @@ You can use the Prometheus metric `loki_ingester_wal_disk_full_failures_total` t ### Backpressure -The WAL also includes a backpressure mechanism to allow a large WAL to be replayed within a smaller memory bound. This is helpful after bad scenarios (i.e. an outage) when a WAL has grown past the point it may be recovered in memory. In this case, the ingester will track the amount of data being replayed and once it's passed the `ingester.wal-replay-memory-ceiling` threshold, will flush to storage. When this happens, it's likely that Loki's attempt to deduplicate chunks via content addressable storage will suffer. We deemed this efficiency loss an acceptable tradeoff considering how it simplifies operation and that it should not occur during regular operation (rollouts, rescheduling) where the WAL can be replayed without triggering this threshold. +The WAL also includes a backpressure mechanism to allow a large WAL to be replayed within a smaller memory bound. This is helpful after bad scenarios (i.e. an outage) when a WAL has grown past the point it may be recovered in memory. In this case, the ingester will track the amount of data being replayed and once it's passed the `ingester.wal-replay-memory-ceiling` threshold, will flush to storage. When this happens, it's likely that the Loki attempt to deduplicate chunks via content addressable storage will suffer. We deemed this efficiency loss an acceptable tradeoff considering how it simplifies operation and that it should not occur during regular operation (rollouts, rescheduling) where the WAL can be replayed without triggering this threshold. ### Metrics @@ -106,7 +106,7 @@ Then you may recreate the (updated) StatefulSet and one-by-one start deleting th #### Scaling Down Using `/flush_shutdown` Endpoint and Lifecycle Hook -1. **StatefulSets for Ordered Scaling Down**: Loki's ingesters should be scaled down one by one, which is efficiently handled by Kubernetes StatefulSets. This ensures an ordered and reliable scaling process, as described in the [Deployment and Scaling Guarantees](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#deployment-and-scaling-guarantees) documentation. +1. **StatefulSets for Ordered Scaling Down**: The Loki ingesters should be scaled down one by one, which is efficiently handled by Kubernetes StatefulSets. This ensures an ordered and reliable scaling process, as described in the [Deployment and Scaling Guarantees](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#deployment-and-scaling-guarantees) documentation. 2. **Using PreStop Lifecycle Hook**: During the Pod scaling down process, the PreStop [lifecycle hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) triggers the `/flush_shutdown` endpoint on the ingester. This action flushes the chunks and removes the ingester from the ring, allowing it to register as unready and become eligible for deletion. @@ -114,7 +114,7 @@ Then you may recreate the (updated) StatefulSet and one-by-one start deleting th 4. **Cleaning Persistent Volumes**: Persistent volumes are automatically cleaned up by leveraging the [enableStatefulSetAutoDeletePVC](https://kubernetes.io/blog/2021/12/16/kubernetes-1-23-statefulset-pvc-auto-deletion/) feature in Kubernetes. -By following the above steps, you can ensure a smooth scaling down process for Loki's ingesters while maintaining data integrity and minimizing potential disruptions. +By following the above steps, you can ensure a smooth scaling down process for the Loki ingesters while maintaining data integrity and minimizing potential disruptions. ### Non-Kubernetes or baremetal deployments diff --git a/docs/sources/operations/zone-ingesters.md b/docs/sources/operations/zone-ingesters.md index ded92065b2255..7467f16ca09f3 100644 --- a/docs/sources/operations/zone-ingesters.md +++ b/docs/sources/operations/zone-ingesters.md @@ -7,7 +7,7 @@ weight: # Zone aware ingesters -Loki's zone aware ingesters are used by Grafana Labs in order to allow for easier rollouts of large Loki deployments. You can think of them as three logical zones, however with some extra Kubernetes configuration you could deploy them in separate zones. +The Loki zone aware ingesters are used by Grafana Labs in order to allow for easier rollouts of large Loki deployments. You can think of them as three logical zones, however with some extra Kubernetes configuration you could deploy them in separate zones. By default, an incoming log stream's logs are replicated to 3 random ingesters. Except in the case of some replica scaling up or down, a given stream will always be replicated to the same 3 ingesters. This means that if one of those ingesters is restarted no data is lost. However two or more ingesters restarting can result in data loss and also impacts the systems ability to ingest logs because of an unhealthy ring status. diff --git a/docs/sources/query/logcli.md b/docs/sources/query/logcli.md index 0ab4deae5c586..9a7d5b18a6d09 100644 --- a/docs/sources/query/logcli.md +++ b/docs/sources/query/logcli.md @@ -1,7 +1,7 @@ --- title: LogCLI menuTItle: -description: Describes LogCLI, Grafana Loki's command-line interface. +description: Describes LogCLI, the Grafana Loki command-line interface. aliases: - ../getting-started/logcli/ - ../tools/logcli/ diff --git a/docs/sources/reference/loki-http-api.md b/docs/sources/reference/loki-http-api.md index 20125fbb6ca21..7ec8155f0cb1f 100644 --- a/docs/sources/reference/loki-http-api.md +++ b/docs/sources/reference/loki-http-api.md @@ -80,7 +80,7 @@ These HTTP endpoints are exposed by the `ingester`, `write`, and `all` component These HTTP endpoints are exposed by the `ruler` component: - [`GET /loki/api/v1/rules`](#list-rule-groups) -- [`GET /loki/api/v1/rules/({namespace}`](#get-rule-groups-by-namespace) +- [`GET /loki/api/v1/rules/{namespace}`](#get-rule-groups-by-namespace) - [`GET /loki/api/v1/rules/{namespace}/{groupName}`](#get-rule-group) - [`POST /loki/api/v1/rules/{namespace}`](#set-rule-group) - [`DELETE /loki/api/v1/rules/{namespace}/{groupName}`](#delete-rule-group) @@ -1048,7 +1048,7 @@ GET /metrics ``` `/metrics` returns exposed Prometheus metrics. See -[Observing Loki]({{< relref "../operations/observability" >}}) +[Observing Loki]({{< relref "../operations/meta-monitoring" >}}) for a list of exported metrics. In microservices mode, the `/metrics` endpoint is exposed by all components. diff --git a/docs/sources/release-notes/v3-1.md b/docs/sources/release-notes/v3-1.md index d67370a4acae2..ab4f0f7c3c999 100644 --- a/docs/sources/release-notes/v3-1.md +++ b/docs/sources/release-notes/v3-1.md @@ -146,7 +146,7 @@ Out of an abundance of caution, we advise that users with Loki or Grafana Enterp - **mixins:** Fix compactor matcher in the loki-deletion dashboard ([#12790](https://github.com/grafana/loki/issues/12790)) ([a03846b](https://github.com/grafana/loki/commit/a03846b4367cbb5a0aa445e539d92ae41e3f481a)). - **mixin:** Mixin generation when cluster label is changed ([#12613](https://github.com/grafana/loki/issues/12613)) ([1ba7a30](https://github.com/grafana/loki/commit/1ba7a303566610363c0c36c87e7bc6bb492dfc93)). - **mixin:** dashboards $__auto fix ([#12707](https://github.com/grafana/loki/issues/12707)) ([91ef72f](https://github.com/grafana/loki/commit/91ef72f742fe1f8621af15d8190c5c0d4d613ab9)). -- **mixins:** Add missing log datasource on loki-deletion ([#13011](https://github.com/grafana/loki/issues/13011)) ([1948899](https://github.com/grafana/loki/commit/1948899999107e7f27f4b9faace64942abcdb41f)). +- **mixins:** Add missing log data source on loki-deletion ([#13011](https://github.com/grafana/loki/issues/13011)) ([1948899](https://github.com/grafana/loki/commit/1948899999107e7f27f4b9faace64942abcdb41f)). - **mixins:** Align loki-writes mixins with loki-reads ([#13022](https://github.com/grafana/loki/issues/13022)) ([757b776](https://github.com/grafana/loki/commit/757b776de39bf0fc0c6d1dd74e4a245d7a99023a)). - **mixins:** Remove unnecessary disk panels for SSD read path ([#13014](https://github.com/grafana/loki/issues/13014)) ([8d9fb68](https://github.com/grafana/loki/commit/8d9fb68ae5d4f26ddc2ae184a1cb6a3b2a2c2127)). - **mixins:** Upgrade old plugin for the loki-operational dashboard. ([#13016](https://github.com/grafana/loki/issues/13016)) ([d3c9cec](https://github.com/grafana/loki/commit/d3c9cec22891b45ed1cb93a9eacc5dad6a117fc5)). diff --git a/docs/sources/send-data/alloy/examples/alloy-kafka-logs.md b/docs/sources/send-data/alloy/examples/alloy-kafka-logs.md index f75cfcc72ac74..11401975b83ac 100644 --- a/docs/sources/send-data/alloy/examples/alloy-kafka-logs.md +++ b/docs/sources/send-data/alloy/examples/alloy-kafka-logs.md @@ -278,7 +278,7 @@ For more information on the `otelcol.processor.batch` configuration, see the [Op ### Write OpenTelemetry logs to Loki -Lastly, we will configure the OpenTelemetry exporter. `otelcol.exporter.otlphttp` accepts telemetry data from other otelcol components and writes them over the network using the OTLP HTTP protocol. We will use this exporter to send the logs to Loki's native OTLP endpoint. +Lastly, we will configure the OpenTelemetry exporter. `otelcol.exporter.otlphttp` accepts telemetry data from other otelcol components and writes them over the network using the OTLP HTTP protocol. We will use this exporter to send the logs to the Loki native OTLP endpoint. Finally, add the following configuration to the `config.alloy` file: ```alloy @@ -392,7 +392,7 @@ Head back to where you started from to continue with the Loki documentation: [Lo For more information on Grafana Alloy, refer to the following resources: - [Grafana Alloy getting started examples](https://grafana.com/docs/alloy/latest/tutorials/) -- [Grafana Alloy common task examples](https://grafana.com/docs/alloy/latest/tasks/) +- [Grafana Alloy common task examples](https://grafana.com/docs/alloy/latest/collect/) - [Grafana Alloy component reference](https://grafana.com/docs/alloy/latest/reference/components/) ## Complete metrics, logs, traces, and profiling example diff --git a/docs/sources/send-data/alloy/examples/alloy-otel-logs.md b/docs/sources/send-data/alloy/examples/alloy-otel-logs.md index 5435dc57435eb..fc7c948bdd4ea 100644 --- a/docs/sources/send-data/alloy/examples/alloy-otel-logs.md +++ b/docs/sources/send-data/alloy/examples/alloy-otel-logs.md @@ -17,7 +17,7 @@ killercoda: Alloy natively supports receiving logs in the OpenTelemetry format. This allows you to send logs from applications instrumented with OpenTelemetry to Alloy, which can then be sent to Loki for storage and visualization in Grafana. In this example, we will make use of 3 Alloy components to achieve this: - **OpenTelemetry Receiver:** This component will receive logs in the OpenTelemetry format via HTTP and gRPC. - **OpenTelemetry Processor:** This component will accept telemetry data from other `otelcol.*` components and place them into batches. Batching improves the compression of data and reduces the number of outgoing network requests required to transmit data. -- **OpenTelemetry Exporter:** This component will accept telemetry data from other `otelcol.*` components and write them over the network using the OTLP HTTP protocol. We will use this exporter to send the logs to Loki's native OTLP endpoint. +- **OpenTelemetry Exporter:** This component will accept telemetry data from other `otelcol.*` components and write them over the network using the OTLP HTTP protocol. We will use this exporter to send the logs to the Loki native OTLP endpoint. @@ -167,7 +167,7 @@ For more information on the `otelcol.processor.batch` configuration, see the [Op ### Export logs to Loki using a OpenTelemetry Exporter -Lastly, we will configure the OpenTelemetry exporter. `otelcol.exporter.otlphttp` accepts telemetry data from other `otelcol` components and writes them over the network using the OTLP HTTP protocol. We will use this exporter to send the logs to Loki's native OTLP endpoint. +Lastly, we will configure the OpenTelemetry exporter. `otelcol.exporter.otlphttp` accepts telemetry data from other `otelcol` components and writes them over the network using the OTLP HTTP protocol. We will use this exporter to send the logs to the Loki native OTLP endpoint. Now add the following configuration to the `config.alloy` file: ```alloy @@ -279,7 +279,7 @@ Head back to where you started from to continue with the Loki documentation: [Lo For more information on Grafana Alloy, refer to the following resources: - [Grafana Alloy getting started examples](https://grafana.com/docs/alloy/latest/tutorials/) -- [Grafana Alloy common task examples](https://grafana.com/docs/alloy/latest/tasks/) +- [Grafana Alloy common task examples](https://grafana.com/docs/alloy/latest/collect/) - [Grafana Alloy component reference](https://grafana.com/docs/alloy/latest/reference/components/) ## Complete metrics, logs, traces, and profiling example diff --git a/docs/sources/send-data/fluentd/_index.md b/docs/sources/send-data/fluentd/_index.md index a42cf4d49b142..c712b0bb166b0 100644 --- a/docs/sources/send-data/fluentd/_index.md +++ b/docs/sources/send-data/fluentd/_index.md @@ -30,7 +30,7 @@ fluent-gem install fluent-plugin-grafana-loki The Docker image `grafana/fluent-plugin-loki:main` contains [default configuration files](https://github.com/grafana/loki/tree/main/clients/cmd/fluentd/docker/conf). By default, fluentd containers use that default configuration. You can instead specify your `fluentd.conf` configuration file with a `FLUENTD_CONF` environment variable. -This image also uses `LOKI_URL`, `LOKI_USERNAME`, and `LOKI_PASSWORD` environment variables to specify the Loki's endpoint, user, and password (you can leave the USERNAME and PASSWORD blank if they're not used). +This image also uses `LOKI_URL`, `LOKI_USERNAME`, and `LOKI_PASSWORD` environment variables to specify the Loki endpoint, user, and password (you can leave the USERNAME and PASSWORD blank if they're not used). This image starts an instance of Fluentd that forwards incoming logs to the specified Loki URL. As an alternate, containerized applications can also use [docker driver plugin]({{< relref "../docker-driver" >}}) to ship logs without needing Fluentd. diff --git a/docs/sources/send-data/otel/native_otlp_vs_loki_exporter.md b/docs/sources/send-data/otel/native_otlp_vs_loki_exporter.md index caad222ba27d9..f2b5fa9f70a65 100644 --- a/docs/sources/send-data/otel/native_otlp_vs_loki_exporter.md +++ b/docs/sources/send-data/otel/native_otlp_vs_loki_exporter.md @@ -103,5 +103,5 @@ Taking the above-ingested log line, let us look at how the querying experience w ## What do you need to do to switch from LokiExporter to native OTel ingestion format? -- Point your OpenTelemetry Collector to Loki's native OTel ingestion endpoint as explained [here](https://grafana.com/docs/loki//send-data/otel/#loki-configuration). +- Point your OpenTelemetry Collector to the Loki native OTel ingestion endpoint as explained [here](https://grafana.com/docs/loki//send-data/otel/#loki-configuration). - Rewrite your LogQL queries in various places, including dashboards, alerts, starred queries in Grafana Explore, etc. to query OTel logs as per the new format. diff --git a/docs/sources/send-data/promtail/_index.md b/docs/sources/send-data/promtail/_index.md index 247328086be37..ec5338d4db1ef 100644 --- a/docs/sources/send-data/promtail/_index.md +++ b/docs/sources/send-data/promtail/_index.md @@ -164,7 +164,7 @@ This endpoint returns 200 when Promtail is up and running, and there's at least ### `GET /metrics` This endpoint returns Promtail metrics for Prometheus. Refer to -[Observing Grafana Loki]({{< relref "../../operations/observability" >}}) for the list +[Observing Grafana Loki]({{< relref "../../operations/meta-monitoring" >}}) for the list of exported metrics. ### Promtail web server config diff --git a/docs/sources/send-data/promtail/stages/limit.md b/docs/sources/send-data/promtail/stages/limit.md index e7a85f13bcd3a..c4612431b0f2a 100644 --- a/docs/sources/send-data/promtail/stages/limit.md +++ b/docs/sources/send-data/promtail/stages/limit.md @@ -14,7 +14,7 @@ The `limit` stage is a rate-limiting stage that throttles logs based on several ## Limit stage schema This pipeline stage places limits on the rate or burst quantity of log lines that Promtail pushes to Loki. -The concept of having distinct burst and rate limits mirrors the approach to limits that can be set for Loki's distributor component: `ingestion_rate_mb` and `ingestion_burst_size_mb`, as defined in [limits_config](https://grafana.com/docs/loki//configure/#limits_config). +The concept of having distinct burst and rate limits mirrors the approach to limits that can be set for the Loki distributor component: `ingestion_rate_mb` and `ingestion_burst_size_mb`, as defined in [limits_config](https://grafana.com/docs/loki//configure/#limits_config). ```yaml limit: diff --git a/docs/sources/setup/install/helm/concepts.md b/docs/sources/setup/install/helm/concepts.md index ceaff0a027c59..23add43de5b4b 100644 --- a/docs/sources/setup/install/helm/concepts.md +++ b/docs/sources/setup/install/helm/concepts.md @@ -28,7 +28,7 @@ By default, the chart installs in [Simple Scalable]({{< relref "./install-scalab The Loki Helm chart does not deploy self-monitoring by default. Loki clusters can be monitored using the meta-monitoring stack, which monitors the logs, metrics, and traces of the Loki cluster. There are two deployment options for this stack, see the installation instructions within [Monitoring]({{< relref "./monitor-and-alert" >}}). -{{< admonition type="Note" >}} +{{< admonition type="note" >}} The meta-monitoring stack replaces the monitoring section of the Loki helm chart which is now **DEPRECATED**. See the [Monitoring]({{< relref "./monitor-and-alert" >}}) section for more information. {{< /admonition >}} @@ -40,7 +40,7 @@ This chart installs the [Loki Canary app]({{< relref "../../../operations/loki-c ## Gateway By default and inspired by Grafana's [Tanka setup](https://github.com/grafana/loki/blob/main/production/ksonnet/loki), the chart -installs the gateway component which is an NGINX that exposes Loki's API and automatically proxies requests to the correct +installs the gateway component which is an NGINX that exposes the Loki API and automatically proxies requests to the correct Loki components (read or write, or single instance in the case of filesystem storage). The gateway must be enabled if an Ingress is required, since the Ingress exposes the gateway only. If the gateway is enabled, Grafana and log shipping agents, such as Promtail, should be configured to use the gateway. diff --git a/docs/sources/setup/install/istio.md b/docs/sources/setup/install/istio.md index 74febca169460..38584be41a1a6 100644 --- a/docs/sources/setup/install/istio.md +++ b/docs/sources/setup/install/istio.md @@ -29,7 +29,7 @@ When you enable istio-injection on the namespace where Loki is running, you need ### Query frontend service -Make the following modifications to the file for Loki's Query Frontend service. +Make the following modifications to the file for the Loki Query Frontend service. 1. Change the name of `grpc` port to `grpclb`. This is used by the grpc load balancing strategy which relies on SRV records. Otherwise the `querier` will not be able to reach the `query-frontend`. See https://github.com/grafana/loki/blob/0116aa61c86fa983ddcbbd5e30a2141d2e89081a/production/ksonnet/loki/common.libsonnet#L19 and @@ -67,7 +67,7 @@ spec: ### Querier service -Make the following modifications to the file for Loki's Querier service. +Make the following modifications to the file for the Loki Querier service. Set the `appProtocol` of the `grpc` service to `tcp` @@ -103,7 +103,7 @@ spec: ### Ingester service and Ingester headless service -Make the following modifications to the file for Loki's Query Ingester and Ingester Headless service. +Make the following modifications to the file for the Loki Query Ingester and Ingester Headless service. Set the `appProtocol` of the `grpc` port to `tcp` @@ -137,7 +137,7 @@ spec: ### Distributor service -Make the following modifications to the file for Loki's Distributor service. +Make the following modifications to the file for the Loki Distributor service. Set the `appProtocol` of the `grpc` port to `tcp` diff --git a/docs/sources/setup/install/tanka.md b/docs/sources/setup/install/tanka.md index baccd14f3a9f7..043a3895892be 100644 --- a/docs/sources/setup/install/tanka.md +++ b/docs/sources/setup/install/tanka.md @@ -49,7 +49,7 @@ Revise the YAML contents of `environments/loki/main.jsonnet`, updating these var - Update the S3 or GCS variable values, depending on your object storage type. See [storage_config](/docs/loki//configuration/#storage_config) for more configuration details. - Remove from the configuration the S3 or GCS object storage variables that are not part of your setup. - Update the Promtail configuration `container_root_path` variable's value to reflect your root path for the Docker daemon. Run `docker info | grep "Root Dir"` to acquire your root path. -- Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on Loki's default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range. +- Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on the Loki default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range. ```jsonnet diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 812544ce43a9c..de3e38a4548b3 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -504,7 +504,7 @@ only in 2.8 and forward releases does the zero value disable retention. The metrics.go log line emitted for every query had an entry called `subqueries` which was intended to represent the amount a query was parallelized on execution. -In the current form it only displayed the count of subqueries generated with Loki's split by time logic and did not include counts for shards. +In the current form it only displayed the count of subqueries generated with the Loki split by time logic and did not include counts for shards. There wasn't a clean way to update subqueries to include sharding information and there is value in knowing the difference between the subqueries generated when we split by time vs sharding factors, especially now that TSDB can do dynamic sharding. diff --git a/docs/sources/setup/upgrade/upgrade-to-6x/index.md b/docs/sources/setup/upgrade/upgrade-to-6x/index.md index ac57907e1f35a..89f9921ccffe4 100644 --- a/docs/sources/setup/upgrade/upgrade-to-6x/index.md +++ b/docs/sources/setup/upgrade/upgrade-to-6x/index.md @@ -48,7 +48,7 @@ Reasons: * The Agent Operator is deprecated. * The dependency on the Prometheus operator is not one we are able to support well. -The [Meta Monitoring Chart](https://github.com/grafana/meta-monitoring-chart) is an improvement over the the previous approach because it allows for installing a clustered Grafana Agent which can send metrics, logs, and traces to Grafana Cloud, or letting you install a monitoring-only local installation of Loki, Mimir, Tempo, and Grafana. +The [Meta Monitoring Chart](https://github.com/grafana/meta-monitoring-chart) is an improvement over the previous approach because it allows for installing a clustered Grafana Agent which can send metrics, logs, and traces to Grafana Cloud, or letting you install a monitoring-only local installation of Loki, Mimir, Tempo, and Grafana. The monitoring sections of this chart still exist but are disabled by default. diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 0e1687d06cc78..a7774c34c3ce6 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -358,7 +358,7 @@ ingester_rf1: # The timeout for an individual flush. Will be retried up to # `flush-op-backoff-retries` times. # CLI flag: -ingester-rf1.flush-op-timeout - [flush_op_timeout: | default = 10m] + [flush_op_timeout: | default = 10s] # Forget about ingesters having heartbeat timestamps older than # `ring.kvstore.heartbeat_timeout`. This is equivalent to clicking on the diff --git a/docs/sources/visualize/grafana.md b/docs/sources/visualize/grafana.md index 9a1ba98c8fd7f..b7da4a6c1bec4 100644 --- a/docs/sources/visualize/grafana.md +++ b/docs/sources/visualize/grafana.md @@ -1,7 +1,7 @@ --- title: Visualize log data menuTitle: Visualize -description: Visualize your log data with Grafana +description: Describes the different ways that you can use Grafana to visualize your log data. aliases: - ../getting-started/grafana/ - ../operations/grafana/ @@ -11,45 +11,56 @@ keywords: - grafana - dashboards --- + # Visualize log data -Modern Grafana versions after 6.3 have built-in support for Grafana Loki and [LogQL](https://grafana.com/docs/loki//query/). +Grafana Loki does not have its own user interface. Most users [install Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/installation/) in order to visualize their log data. Grafana versions after 6.3 have built-in support for Grafana Loki and [LogQL](https://grafana.com/docs/loki//query/). + +There are several different options for how to visualize your log data in Grafana: + +- [Explore Logs](https://grafana.com/docs/grafana-cloud/visualizations/simplified-exploration/logs/) lets you explore logs from your Loki data source without writing LogQL queries. Explore Logs is now available in public preview. +- [Grafana Explore](https://grafana.com/docs/grafana/latest/explore/logs-integration/) helps you build and iterate on queries written in LogQL. Once you have a query that finds the data you're looking for, you can consider using your query in a Grafana dashboard. +- [Loki Mixins](https://grafana.com/docs/loki/latest/operations/observability/#mixins) include a pre-built set of dashboards, recording rules, and alerts for monitoring Loki. +- [Grafana Dashboards](https://grafana.com/docs/grafana/latest/dashboards/) let you query, transform, visualize, and understand your log data. You can create your own custom dashboards, or import and modify public dashboards shared by the community. + +## Explore Logs + +Explore Logs lets you automatically visualize and explore logs. Explore Logs makes assumptions about what data you might want to see to help you quickly get started viewing your logs without having to learn LogQL and write queries. -## Using Explore +If you are a Grafana Cloud user, you can access Explore Logs in the Grafana Cloud main navigation menu. If you are not a Grafana Cloud user, you can install the [Explore Logs plugin](https://grafana.com/docs/grafana-cloud/visualizations/simplified-exploration/logs/access/). For more information, refer to the [Explore Logs documentation](https://grafana.com/docs/grafana-cloud/visualizations/simplified-exploration/logs/). -1. Log into your Grafana instance. If this is your first time running - Grafana, the username and password are both defaulted to `admin`. -1. In Grafana, go to `Connections` > `Data Sources` via the cog icon on the - left sidebar. -1. Click the big + Add a new data source button. +## Grafana Explore + +[Grafana Explore](https://grafana.com/docs/grafana/latest/explore/) helps you build and iterate on a LogQL query outside of the dashboard user interface. If you just want to explore your data and do not want to create a dashboard, then Explore makes this much easier. + +1. Log into your Grafana instance. If this is your first time running Grafana, the username and password are both defaulted to `admin`. +1. In the Grafana main menu, select **Connections** > **Data source**. +1. Click the **+ Add new data source** button. 1. Search for, or choose Loki from the list. -1. The http URL field should be the address of your Loki server. For example, - when running locally or with Docker using port mapping, the address is - likely `http://localhost:3100`. When running with docker-compose or - Kubernetes, the address is likely `http://loki:3100`.\ - When running Grafana (with Docker) and trying to connect to a locally built Loki instance, the address (for the URL field) is:\ - On Mac: `docker.for.mac.localhost` \ +1. On the **Settings** tab, the **URL** field should be the address of your Loki server. +For example,when running locally or with Docker using port mapping, the address is likely `http://localhost:3100`. +When running with docker-compose or Kubernetes, the address is likely `http://loki:3100`. +When running Grafana (with Docker) and trying to connect to a locally built Loki instance, the address (for the URL field) is: + On Mac: `docker.for.mac.localhost` On Windows: `docker.for.win.localhost` -1. To see the logs, click Explore on the sidebar, select the Loki - data source in the top-left dropdown, and then choose a log stream using the - Log labels button. -1. Learn more about querying by reading about Loki's query language [LogQL]({{< relref "../query/_index.md" >}}). +1. To view your logs, click **Explore** in the main menu. +1. Select the Loki datasource in the top-left menu. +1. You can click **Kick start your query** to select from a list of common queries, or use the **Label filters** to start choosing labels that you want to query. For more information about the Loki query language, refer to the [LogQL section](https://grafana.com/docs/loki//query/). -If you would like to see an example of this live, you can try [Grafana Play's Explore feature](https://play.grafana.org/explore?schemaVersion=1&panes=%7B%22v1d%22:%7B%22datasource%22:%22ac4000ca-1959-45f5-aa45-2bd0898f7026%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bagent%3D%5C%22promtail%5C%22%7D%20%7C%3D%20%60%60%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22ac4000ca-1959-45f5-aa45-2bd0898f7026%22%7D,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1) +If you would like to see an example of this live, you can try [Grafana Play's Explore feature](https://play.grafana.org/explore?schemaVersion=1&panes=%7B%22v1d%22:%7B%22datasource%22:%22ac4000ca-1959-45f5-aa45-2bd0898f7026%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bagent%3D%5C%22promtail%5C%22%7D%20%7C%3D%20%60%60%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22ac4000ca-1959-45f5-aa45-2bd0898f7026%22%7D,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1). -Read more about Grafana's Explore feature in the -[Grafana documentation](http://docs.grafana.org/features/explore) and on how to -search and filter for logs with Loki. +Learn more about the Grafana Explore feature in the [Grafana documentation](https://grafana.com/docs/grafana/latest/explore/logs-integration/). -## Using Grafana Dashboards +## Loki mixins -Because Loki can be used as a built-in data source above, we can use LogQL queries based on that datasource -to build complex visualizations that persist on Grafana dashboards. +The Loki mixin provides a set of Grafana dashboards, Prometheus recording rules and alerts for monitoring Loki itself. For instructions on how to install the Loki mixins, refer to the [installation topic](https://grafana.com/docs/loki//operations/meta-monitoring/mixins/). + +## Using Grafana dashboards {{< docs/play title="Loki Example Grafana Dashboard" url="https://play.grafana.org/d/T512JVH7z/" >}} -Read more about how to build Grafana Dashboards in [build your first dashbboard](https://grafana.com/docs/grafana/latest/getting-started/build-first-dashboard/) +Because Loki can be used as a built-in data source, you can use LogQL queries based on that data source to build complex visualizations that persist on Grafana dashboards. + +To configure Loki as a data source via provisioning, refer to the documentation for [Loki data source](https://grafana.com/docs/grafana/latest/datasources/loki/#configure-the-datasource-with-provisioning). -To configure Loki as a data source via provisioning, see [Configuring Grafana via -Provisioning](http://docs.grafana.org/features/datasources/loki/#configure-the-datasource-with-provisioning). -Set the URL in the provisioning. +Read more about how to build Grafana Dashboards in [build your first dashboard](https://grafana.com/docs/grafana/latest/getting-started/build-first-dashboard/). diff --git a/loki-build-image/Dockerfile b/loki-build-image/Dockerfile index 6b49181606c48..05dc022441609 100644 --- a/loki-build-image/Dockerfile +++ b/loki-build-image/Dockerfile @@ -6,7 +6,7 @@ # on how to publish a new build image. ARG GO_VERSION=1.22 # Install helm (https://helm.sh/) and helm-docs (https://github.com/norwoodj/helm-docs) for generating Helm Chart reference. -FROM golang:${GO_VERSION}-bookworm as helm +FROM golang:${GO_VERSION}-bookworm AS helm ARG TARGETARCH ARG HELM_VER="v3.2.3" RUN curl -L "https://get.helm.sh/helm-${HELM_VER}-linux-$TARGETARCH.tar.gz" | tar zx && \ @@ -15,7 +15,7 @@ RUN BIN=$([ "$TARGETARCH" = "arm64" ] && echo "helm-docs_Linux_arm64" || echo "h curl -L "https://github.com/norwoodj/helm-docs/releases/download/v1.11.2/$BIN.tar.gz" | tar zx && \ install -t /usr/local/bin helm-docs -FROM alpine:3.18.6 as lychee +FROM alpine:3.20.2 AS lychee ARG TARGETARCH ARG LYCHEE_VER="0.7.0" RUN apk add --no-cache curl && \ @@ -24,21 +24,21 @@ RUN apk add --no-cache curl && \ mv /tmp/lychee /usr/bin/lychee && \ rm -rf "/tmp/linux-$TARGETARCH" /tmp/lychee-$LYCHEE_VER.tgz -FROM alpine:3.18.6 as golangci +FROM alpine:3.20.2 AS golangci RUN apk add --no-cache curl && \ cd / && \ curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.55.1 -FROM alpine:3.18.6 as buf +FROM alpine:3.20.2 AS buf ARG TARGETOS RUN apk add --no-cache curl && \ curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.4.0/buf-$TARGETOS-$(uname -m)" -o "/usr/bin/buf" && \ chmod +x "/usr/bin/buf" -FROM alpine:3.18.6 as docker +FROM alpine:3.20.2 AS docker RUN apk add --no-cache docker-cli docker-cli-buildx -FROM golang:${GO_VERSION}-bookworm as drone +FROM golang:${GO_VERSION}-bookworm AS drone ARG TARGETARCH RUN curl -L "https://github.com/drone/drone-cli/releases/download/v1.7.0/drone_linux_$TARGETARCH".tar.gz | tar zx && \ install -t /usr/local/bin drone @@ -48,33 +48,33 @@ RUN curl -L "https://github.com/drone/drone-cli/releases/download/v1.7.0/drone_l # Error: # github.com/fatih/faillint@v1.5.0 requires golang.org/x/tools@v0.0.0-20200207224406-61798d64f025 # (not golang.org/x/tools@v0.0.0-20190918214920-58d531046acd from golang.org/x/tools/cmd/goyacc@58d531046acdc757f177387bc1725bfa79895d69) -FROM golang:${GO_VERSION}-bookworm as faillint +FROM golang:${GO_VERSION}-bookworm AS faillint RUN GO111MODULE=on go install github.com/fatih/faillint@v1.12.0 RUN GO111MODULE=on go install golang.org/x/tools/cmd/goimports@v0.7.0 -FROM golang:${GO_VERSION}-bookworm as delve +FROM golang:${GO_VERSION}-bookworm AS delve RUN GO111MODULE=on go install github.com/go-delve/delve/cmd/dlv@latest # Install ghr used to push binaries and template the release # This collides with the version of go tools used in the base image, thus we install it in its own image and copy it over. -FROM golang:${GO_VERSION}-bookworm as ghr +FROM golang:${GO_VERSION}-bookworm AS ghr RUN GO111MODULE=on go install github.com/tcnksm/ghr@9349474 # Install nfpm (https://nfpm.goreleaser.com) for creating .deb and .rpm packages. -FROM golang:${GO_VERSION}-bookworm as nfpm +FROM golang:${GO_VERSION}-bookworm AS nfpm RUN GO111MODULE=on go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.11.3 # Install gotestsum -FROM golang:${GO_VERSION}-bookworm as gotestsum +FROM golang:${GO_VERSION}-bookworm AS gotestsum RUN GO111MODULE=on go install gotest.tools/gotestsum@v1.8.2 # Install tools used to compile jsonnet. -FROM golang:${GO_VERSION}-bookworm as jsonnet +FROM golang:${GO_VERSION}-bookworm AS jsonnet RUN GO111MODULE=on go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1 RUN GO111MODULE=on go install github.com/monitoring-mixins/mixtool/cmd/mixtool@16dc166166d91e93475b86b9355a4faed2400c18 RUN GO111MODULE=on go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0 -FROM aquasec/trivy as trivy +FROM aquasec/trivy AS trivy FROM golang:${GO_VERSION}-bookworm RUN apt-get update && \ diff --git a/loki-build-image/README.md b/loki-build-image/README.md index dd757886b7b88..37769c9411cee 100644 --- a/loki-build-image/README.md +++ b/loki-build-image/README.md @@ -2,6 +2,10 @@ ## Versions +### 0.33.5 + +- Update to alpine 3.20.2 + ### 0.33.4 - Update to go 1.22.5 diff --git a/operator/docs/lokistack/sop.md b/operator/docs/lokistack/sop.md index 1ae656e1d5e85..51048360f3374 100644 --- a/operator/docs/lokistack/sop.md +++ b/operator/docs/lokistack/sop.md @@ -309,6 +309,38 @@ The query queue is currently under high load. - Increase the number of queriers +## Loki Discarded Samples Warning + +### Impact + +Loki is discarding samples (log entries) because they fail validation. This alert only fires for errors that are not retryable. This means that the discarded samples are lost. + +### Summary + +Loki can reject log entries (samples) during submission when they fail validation. This happens on a per-stream basis, so only the specific samples or streams failing validation are lost. + +The possible validation errors are documented in the [Loki documentation](https://grafana.com/docs/loki/latest/operations/request-validation-rate-limits/#validation-errors). This alert only fires for the validation errors that are not retryable, which means that discarded samples are permanently lost. + +The alerting can only show the affected Loki tenant. Since Loki 3.1.0 more detailed information about the affected streams is provided in an error message emitted by the distributor component. + +This information can be used to pinpoint the application sending the offending logs. For some of the validations there are configuration parameters that can be tuned in LokiStack's `limits` structure, if the messages should be accepted. Usually it is recommended to fix the issue either on the emitting application (if possible) or by changing collector configuration to fix non-compliant messages before sending them to Loki. + +### Severity + +`Warning` + +### Access Required + +- Console access to the cluster +- View access in the namespace where the LokiStack is deployed + - OpenShift + - `openshift-logging` (LokiStack) + +### Steps + +- View detailed log output from the Loki distributors to identify affected streams +- Decide on further steps depending on log source and validation error + ## Lokistack Storage Schema Warning ### Impact @@ -332,4 +364,4 @@ The schema configuration does not contain the most recent schema version and nee ### Steps -- Add a new object storage schema V13 with a future EffectiveDate \ No newline at end of file +- Add a new object storage schema V13 with a future EffectiveDate diff --git a/operator/internal/manifests/internal/alerts/prometheus-alerts.yaml b/operator/internal/manifests/internal/alerts/prometheus-alerts.yaml index 15cc42462f158..799c280d3a9e8 100644 --- a/operator/internal/manifests/internal/alerts/prometheus-alerts.yaml +++ b/operator/internal/manifests/internal/alerts/prometheus-alerts.yaml @@ -175,6 +175,24 @@ groups: for: 15m labels: severity: warning + - alert: LokiDiscardedSamplesWarning + annotations: + message: |- + Loki in namespace {{ $labels.namespace }} is discarding samples in the "{{ $labels.tenant }}" tenant during ingestion. + Samples are discarded because of "{{ $labels.reason }}" at a rate of {{ .Value | humanize }} samples per second. + summary: Loki is discarding samples during ingestion because they fail validation. + runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning" + expr: | + sum by(namespace, tenant, reason) ( + irate(loki_discarded_samples_total{ + reason!="rate_limited", + reason!="per_stream_rate_limit", + reason!="stream_limit"}[2m]) + ) + > 0 + for: 15m + labels: + severity: warning - alert: LokistackSchemaUpgradesRequired annotations: message: |- diff --git a/operator/internal/manifests/internal/alerts/testdata/test.yaml b/operator/internal/manifests/internal/alerts/testdata/test.yaml index a4d8bec8a6a4c..d60e3befa0023 100644 --- a/operator/internal/manifests/internal/alerts/testdata/test.yaml +++ b/operator/internal/manifests/internal/alerts/testdata/test.yaml @@ -63,6 +63,9 @@ tests: - series: 'loki_logql_querystats_latency_seconds_bucket{namespace="my-ns", job="querier", route="my-route", le="+Inf"}' values: '0+100x20' + - series: 'loki_discarded_samples_total{namespace="my-ns", tenant="application", reason="line_too_long"}' + values: '0x5 0+120x25 3000' + alert_rule_test: - eval_time: 16m alertname: LokiRequestErrors @@ -177,3 +180,17 @@ tests: summary: "The read path has high volume of queries, causing longer response times." message: "The read path is experiencing high load." runbook_url: "[[ .RunbookURL ]]#Loki-Read-Path-High-Load" + - eval_time: 22m + alertname: LokiDiscardedSamplesWarning + exp_alerts: + - exp_labels: + namespace: my-ns + tenant: application + severity: warning + reason: line_too_long + exp_annotations: + message: |- + Loki in namespace my-ns is discarding samples in the "application" tenant during ingestion. + Samples are discarded because of "line_too_long" at a rate of 2 samples per second. + summary: Loki is discarding samples during ingestion because they fail validation. + runbook_url: "[[ .RunbookURL]]#Loki-Discarded-Samples-Warning" diff --git a/pkg/bloombuild/builder/builder.go b/pkg/bloombuild/builder/builder.go index f02afe30a2891..045f96bc7f591 100644 --- a/pkg/bloombuild/builder/builder.go +++ b/pkg/bloombuild/builder/builder.go @@ -336,7 +336,7 @@ func (b *Builder) processTask( // Fetch blocks that aren't up to date but are in the desired fingerprint range // to try and accelerate bloom creation. level.Debug(logger).Log("msg", "loading series and blocks for gap", "blocks", len(gap.Blocks)) - seriesItr, blocksIter, err := b.loadWorkForGap(ctx, task.Table, tenant, task.TSDB, gap) + seriesItr, blocksIter, err := b.loadWorkForGap(ctx, task.Table, gap) if err != nil { level.Error(logger).Log("msg", "failed to get series and blocks", "err", err) return nil, fmt.Errorf("failed to get series and blocks: %w", err) @@ -455,15 +455,9 @@ func (b *Builder) processTask( func (b *Builder) loadWorkForGap( ctx context.Context, table config.DayTable, - tenant string, - id tsdb.Identifier, - gap protos.GapWithBlocks, + gap protos.Gap, ) (iter.Iterator[*v1.Series], iter.CloseResetIterator[*v1.SeriesWithBlooms], error) { - // load a series iterator for the gap - seriesItr, err := b.tsdbStore.LoadTSDB(ctx, table, tenant, id, gap.Bounds) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load tsdb") - } + seriesItr := iter.NewCancelableIter[*v1.Series](ctx, iter.NewSliceIter[*v1.Series](gap.Series)) // load a blocks iterator for the gap fetcher, err := b.bloomStore.Fetcher(table.ModelTime()) diff --git a/pkg/bloombuild/common/tsdb.go b/pkg/bloombuild/common/tsdb.go index 8082a8b319a47..a2e22529523b2 100644 --- a/pkg/bloombuild/common/tsdb.go +++ b/pkg/bloombuild/common/tsdb.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" @@ -30,6 +29,11 @@ const ( gzipExtension = ".gz" ) +type ClosableForSeries interface { + sharding.ForSeries + Close() error +} + type TSDBStore interface { UsersForPeriod(ctx context.Context, table config.DayTable) ([]string, error) ResolveTSDBs(ctx context.Context, table config.DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) @@ -38,8 +42,7 @@ type TSDBStore interface { table config.DayTable, tenant string, id tsdb.Identifier, - bounds v1.FingerprintBounds, - ) (iter.Iterator[*v1.Series], error) + ) (ClosableForSeries, error) } // BloomTSDBStore is a wrapper around the storage.Client interface which @@ -90,8 +93,7 @@ func (b *BloomTSDBStore) LoadTSDB( table config.DayTable, tenant string, id tsdb.Identifier, - bounds v1.FingerprintBounds, -) (iter.Iterator[*v1.Series], error) { +) (ClosableForSeries, error) { withCompression := id.Name() + gzipExtension data, err := b.storage.GetUserFile(ctx, table.Addr(), tenant, withCompression) @@ -118,13 +120,8 @@ func (b *BloomTSDBStore) LoadTSDB( } idx := tsdb.NewTSDBIndex(reader) - defer func() { - if err := idx.Close(); err != nil { - level.Error(b.logger).Log("msg", "failed to close index", "err", err) - } - }() - return NewTSDBSeriesIter(ctx, tenant, idx, bounds) + return idx, nil } func NewTSDBSeriesIter(ctx context.Context, user string, f sharding.ForSeries, bounds v1.FingerprintBounds) (iter.Iterator[*v1.Series], error) { @@ -251,12 +248,11 @@ func (s *TSDBStores) LoadTSDB( table config.DayTable, tenant string, id tsdb.Identifier, - bounds v1.FingerprintBounds, -) (iter.Iterator[*v1.Series], error) { +) (ClosableForSeries, error) { store, err := s.storeForPeriod(table.DayTime) if err != nil { return nil, err } - return store.LoadTSDB(ctx, table, tenant, id, bounds) + return store.LoadTSDB(ctx, table, tenant, id) } diff --git a/pkg/bloombuild/planner/planner.go b/pkg/bloombuild/planner/planner.go index 285795a8327d2..f65fdf59c9acb 100644 --- a/pkg/bloombuild/planner/planner.go +++ b/pkg/bloombuild/planner/planner.go @@ -365,6 +365,29 @@ func (p *Planner) computeTasks( return nil, nil, fmt.Errorf("failed to delete outdated metas during planning: %w", err) } + // Resolve TSDBs + tsdbs, err := p.tsdbStore.ResolveTSDBs(ctx, table, tenant) + if err != nil { + level.Error(logger).Log("msg", "failed to resolve tsdbs", "err", err) + return nil, nil, fmt.Errorf("failed to resolve tsdbs: %w", err) + } + + if len(tsdbs) == 0 { + return nil, metas, nil + } + + openTSDBs, err := openAllTSDBs(ctx, table, tenant, p.tsdbStore, tsdbs) + if err != nil { + return nil, nil, fmt.Errorf("failed to open all tsdbs: %w", err) + } + defer func() { + for idx, reader := range openTSDBs { + if err := reader.Close(); err != nil { + level.Error(logger).Log("msg", "failed to close index", "err", err, "tsdb", idx.Name()) + } + } + }() + for _, ownershipRange := range ownershipRanges { logger := log.With(logger, "ownership", ownershipRange.String()) @@ -372,7 +395,7 @@ func (p *Planner) computeTasks( metasInBounds := bloomshipper.FilterMetasOverlappingBounds(metas, ownershipRange) // Find gaps in the TSDBs for this tenant/table - gaps, err := p.findOutdatedGaps(ctx, tenant, table, ownershipRange, metasInBounds, logger) + gaps, err := p.findOutdatedGaps(ctx, tenant, openTSDBs, ownershipRange, metasInBounds, logger) if err != nil { level.Error(logger).Log("msg", "failed to find outdated gaps", "err", err) continue @@ -453,6 +476,26 @@ func (p *Planner) processTenantTaskResults( return tasksSucceed, nil } +func openAllTSDBs( + ctx context.Context, + table config.DayTable, + tenant string, + store common.TSDBStore, + tsdbs []tsdb.SingleTenantTSDBIdentifier, +) (map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries, error) { + openTSDBs := make(map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries, len(tsdbs)) + for _, idx := range tsdbs { + tsdb, err := store.LoadTSDB(ctx, table, tenant, idx) + if err != nil { + return nil, fmt.Errorf("failed to load tsdb: %w", err) + } + + openTSDBs[idx] = tsdb + } + + return openTSDBs, nil +} + // deleteOutdatedMetasAndBlocks filters out the outdated metas from the `metas` argument and deletes them from the store. // It returns the up-to-date metas from the `metas` argument. func (p *Planner) deleteOutdatedMetasAndBlocks( @@ -655,28 +698,17 @@ func (p *Planner) tenants(ctx context.Context, table config.DayTable) (*iter.Sli // This is a performance optimization to avoid expensive re-reindexing type blockPlan struct { tsdb tsdb.SingleTenantTSDBIdentifier - gaps []protos.GapWithBlocks + gaps []protos.Gap } func (p *Planner) findOutdatedGaps( ctx context.Context, tenant string, - table config.DayTable, + tsdbs map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries, ownershipRange v1.FingerprintBounds, metas []bloomshipper.Meta, logger log.Logger, ) ([]blockPlan, error) { - // Resolve TSDBs - tsdbs, err := p.tsdbStore.ResolveTSDBs(ctx, table, tenant) - if err != nil { - level.Error(logger).Log("msg", "failed to resolve tsdbs", "err", err) - return nil, fmt.Errorf("failed to resolve tsdbs: %w", err) - } - - if len(tsdbs) == 0 { - return nil, nil - } - // Determine which TSDBs have gaps in the ownership range and need to // be processed. tsdbsWithGaps, err := gapsBetweenTSDBsAndMetas(ownershipRange, tsdbs, metas) @@ -690,7 +722,7 @@ func (p *Planner) findOutdatedGaps( return nil, nil } - work, err := blockPlansForGaps(tsdbsWithGaps, metas) + work, err := blockPlansForGaps(ctx, tenant, tsdbsWithGaps, metas) if err != nil { level.Error(logger).Log("msg", "failed to create plan", "err", err) return nil, fmt.Errorf("failed to create plan: %w", err) @@ -701,18 +733,19 @@ func (p *Planner) findOutdatedGaps( // Used to signal the gaps that need to be populated for a tsdb type tsdbGaps struct { - tsdb tsdb.SingleTenantTSDBIdentifier - gaps []v1.FingerprintBounds + tsdbIdentifier tsdb.SingleTenantTSDBIdentifier + tsdb common.ClosableForSeries + gaps []v1.FingerprintBounds } // gapsBetweenTSDBsAndMetas returns if the metas are up-to-date with the TSDBs. This is determined by asserting // that for each TSDB, there are metas covering the entire ownership range which were generated from that specific TSDB. func gapsBetweenTSDBsAndMetas( ownershipRange v1.FingerprintBounds, - tsdbs []tsdb.SingleTenantTSDBIdentifier, + tsdbs map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries, metas []bloomshipper.Meta, ) (res []tsdbGaps, err error) { - for _, db := range tsdbs { + for db, tsdb := range tsdbs { id := db.Name() relevantMetas := make([]v1.FingerprintBounds, 0, len(metas)) @@ -731,8 +764,9 @@ func gapsBetweenTSDBsAndMetas( if len(gaps) > 0 { res = append(res, tsdbGaps{ - tsdb: db, - gaps: gaps, + tsdbIdentifier: db, + tsdb: tsdb, + gaps: gaps, }) } } @@ -743,22 +777,35 @@ func gapsBetweenTSDBsAndMetas( // blockPlansForGaps groups tsdb gaps we wish to fill with overlapping but out of date blocks. // This allows us to expedite bloom generation by using existing blocks to fill in the gaps // since many will contain the same chunks. -func blockPlansForGaps(tsdbs []tsdbGaps, metas []bloomshipper.Meta) ([]blockPlan, error) { +func blockPlansForGaps( + ctx context.Context, + tenant string, + tsdbs []tsdbGaps, + metas []bloomshipper.Meta, +) ([]blockPlan, error) { plans := make([]blockPlan, 0, len(tsdbs)) for _, idx := range tsdbs { plan := blockPlan{ - tsdb: idx.tsdb, - gaps: make([]protos.GapWithBlocks, 0, len(idx.gaps)), + tsdb: idx.tsdbIdentifier, + gaps: make([]protos.Gap, 0, len(idx.gaps)), } for _, gap := range idx.gaps { - planGap := protos.GapWithBlocks{ + planGap := protos.Gap{ Bounds: gap, } - for _, meta := range metas { + seriesItr, err := common.NewTSDBSeriesIter(ctx, tenant, idx.tsdb, gap) + if err != nil { + return nil, fmt.Errorf("failed to load series from TSDB for gap (%s): %w", gap.String(), err) + } + planGap.Series, err = iter.Collect(seriesItr) + if err != nil { + return nil, fmt.Errorf("failed to collect series: %w", err) + } + for _, meta := range metas { if meta.Bounds.Intersection(gap) == nil { // this meta doesn't overlap the gap, skip continue diff --git a/pkg/bloombuild/planner/planner_test.go b/pkg/bloombuild/planner/planner_test.go index ca5c1d0c15b09..26a3b880c1d6a 100644 --- a/pkg/bloombuild/planner/planner_test.go +++ b/pkg/bloombuild/planner/planner_test.go @@ -16,10 +16,12 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" "go.uber.org/atomic" "google.golang.org/grpc" + "github.com/grafana/loki/v3/pkg/bloombuild/common" "github.com/grafana/loki/v3/pkg/bloombuild/protos" "github.com/grafana/loki/v3/pkg/chunkenc" iter "github.com/grafana/loki/v3/pkg/iter/v2" @@ -31,6 +33,7 @@ import ( "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" bloomshipperconfig "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/v3/pkg/storage/types" "github.com/grafana/loki/v3/pkg/util/mempool" ) @@ -68,14 +71,16 @@ func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { err bool exp []tsdbGaps ownershipRange v1.FingerprintBounds - tsdbs []tsdb.SingleTenantTSDBIdentifier + tsdbs map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries metas []bloomshipper.Meta }{ { desc: "non-overlapping tsdbs and metas", err: true, ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + tsdbs: map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries{ + tsdbID(0): nil, + }, metas: []bloomshipper.Meta{ genMeta(11, 20, []int{0}, nil), }, @@ -83,13 +88,15 @@ func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { { desc: "single tsdb", ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0)}, + tsdbs: map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries{ + tsdbID(0): nil, + }, metas: []bloomshipper.Meta{ genMeta(4, 8, []int{0}, nil), }, exp: []tsdbGaps{ { - tsdb: tsdbID(0), + tsdbIdentifier: tsdbID(0), gaps: []v1.FingerprintBounds{ v1.NewBounds(0, 3), v1.NewBounds(9, 10), @@ -100,20 +107,23 @@ func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { { desc: "multiple tsdbs with separate blocks", ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, + tsdbs: map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries{ + tsdbID(0): nil, + tsdbID(1): nil, + }, metas: []bloomshipper.Meta{ genMeta(0, 5, []int{0}, nil), genMeta(6, 10, []int{1}, nil), }, exp: []tsdbGaps{ { - tsdb: tsdbID(0), + tsdbIdentifier: tsdbID(0), gaps: []v1.FingerprintBounds{ v1.NewBounds(6, 10), }, }, { - tsdb: tsdbID(1), + tsdbIdentifier: tsdbID(1), gaps: []v1.FingerprintBounds{ v1.NewBounds(0, 5), }, @@ -123,20 +133,23 @@ func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { { desc: "multiple tsdbs with the same blocks", ownershipRange: v1.NewBounds(0, 10), - tsdbs: []tsdb.SingleTenantTSDBIdentifier{tsdbID(0), tsdbID(1)}, + tsdbs: map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries{ + tsdbID(0): nil, + tsdbID(1): nil, + }, metas: []bloomshipper.Meta{ genMeta(0, 5, []int{0, 1}, nil), genMeta(6, 8, []int{1}, nil), }, exp: []tsdbGaps{ { - tsdb: tsdbID(0), + tsdbIdentifier: tsdbID(0), gaps: []v1.FingerprintBounds{ v1.NewBounds(6, 10), }, }, { - tsdb: tsdbID(1), + tsdbIdentifier: tsdbID(1), gaps: []v1.FingerprintBounds{ v1.NewBounds(9, 10), }, @@ -150,7 +163,7 @@ func Test_gapsBetweenTSDBsAndMetas(t *testing.T) { require.Error(t, err) return } - require.Equal(t, tc.exp, gaps) + require.ElementsMatch(t, tc.exp, gaps) }) } } @@ -220,9 +233,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ { Bounds: v1.NewBounds(0, 10), + Series: genSeries(v1.NewBounds(0, 10)), }, }, }, @@ -238,9 +252,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ { Bounds: v1.NewBounds(0, 10), + Series: genSeries(v1.NewBounds(0, 10)), Blocks: []bloomshipper.BlockRef{genBlockRef(9, 20)}, }, }, @@ -261,9 +276,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ { Bounds: v1.NewBounds(0, 8), + Series: genSeries(v1.NewBounds(0, 8)), }, }, }, @@ -280,9 +296,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ { Bounds: v1.NewBounds(0, 8), + Series: genSeries(v1.NewBounds(0, 8)), Blocks: []bloomshipper.BlockRef{genBlockRef(5, 20)}, }, }, @@ -306,14 +323,16 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ // tsdb (id=0) can source chunks from the blocks built from tsdb (id=1) { Bounds: v1.NewBounds(3, 5), + Series: genSeries(v1.NewBounds(3, 5)), Blocks: []bloomshipper.BlockRef{genBlockRef(3, 5)}, }, { Bounds: v1.NewBounds(9, 10), + Series: genSeries(v1.NewBounds(9, 10)), Blocks: []bloomshipper.BlockRef{genBlockRef(8, 10)}, }, }, @@ -321,9 +340,10 @@ func Test_blockPlansForGaps(t *testing.T) { // tsdb (id=1) can source chunks from the blocks built from tsdb (id=0) { tsdb: tsdbID(1), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ { Bounds: v1.NewBounds(0, 2), + Series: genSeries(v1.NewBounds(0, 2)), Blocks: []bloomshipper.BlockRef{ genBlockRef(0, 1), genBlockRef(1, 2), @@ -331,6 +351,7 @@ func Test_blockPlansForGaps(t *testing.T) { }, { Bounds: v1.NewBounds(6, 7), + Series: genSeries(v1.NewBounds(6, 7)), Blocks: []bloomshipper.BlockRef{genBlockRef(6, 8)}, }, }, @@ -354,9 +375,10 @@ func Test_blockPlansForGaps(t *testing.T) { exp: []blockPlan{ { tsdb: tsdbID(0), - gaps: []protos.GapWithBlocks{ + gaps: []protos.Gap{ { Bounds: v1.NewBounds(0, 10), + Series: genSeries(v1.NewBounds(0, 10)), Blocks: []bloomshipper.BlockRef{ genBlockRef(1, 4), genBlockRef(5, 10), @@ -369,20 +391,86 @@ func Test_blockPlansForGaps(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { + // We add series spanning the whole FP ownership range + tsdbs := make(map[tsdb.SingleTenantTSDBIdentifier]common.ClosableForSeries) + for _, id := range tc.tsdbs { + tsdbs[id] = newFakeForSeries(genSeries(tc.ownershipRange)) + } + // we reuse the gapsBetweenTSDBsAndMetas function to generate the gaps as this function is tested // separately and it's used to generate input in our regular code path (easier to write tests this way). - gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tc.tsdbs, tc.metas) + gaps, err := gapsBetweenTSDBsAndMetas(tc.ownershipRange, tsdbs, tc.metas) require.NoError(t, err) - plans, err := blockPlansForGaps(gaps, tc.metas) + plans, err := blockPlansForGaps( + context.Background(), + "fakeTenant", + gaps, + tc.metas, + ) if tc.err { require.Error(t, err) return } require.Equal(t, tc.exp, plans) + }) + } +} +func genSeries(bounds v1.FingerprintBounds) []*v1.Series { + series := make([]*v1.Series, 0, int(bounds.Max-bounds.Min+1)) + for i := bounds.Min; i <= bounds.Max; i++ { + series = append(series, &v1.Series{ + Fingerprint: i, + Chunks: v1.ChunkRefs{ + { + From: 0, + Through: 1, + Checksum: 1, + }, + }, }) } + return series +} + +type fakeForSeries struct { + series []*v1.Series +} + +func newFakeForSeries(series []*v1.Series) *fakeForSeries { + return &fakeForSeries{ + series: series, + } +} + +func (f fakeForSeries) ForSeries(_ context.Context, _ string, ff index.FingerprintFilter, _ model.Time, _ model.Time, fn func(labels.Labels, model.Fingerprint, []index.ChunkMeta) (stop bool), _ ...*labels.Matcher) error { + overlapping := make([]*v1.Series, 0, len(f.series)) + for _, s := range f.series { + if ff.Match(s.Fingerprint) { + overlapping = append(overlapping, s) + } + } + + for _, s := range overlapping { + chunks := make([]index.ChunkMeta, 0, len(s.Chunks)) + for _, c := range s.Chunks { + chunks = append(chunks, index.ChunkMeta{ + MinTime: int64(c.From), + MaxTime: int64(c.Through), + Checksum: c.Checksum, + }) + } + + if fn(labels.EmptyLabels(), s.Fingerprint, chunks) { + break + } + } + return nil +} + +func (f fakeForSeries) Close() error { + return nil } func createTasks(n int, resultsCh chan *protos.TaskResult) []*QueueTask { diff --git a/pkg/bloombuild/protos/compat.go b/pkg/bloombuild/protos/compat.go index ad7c492cc5fc9..468278e77dbea 100644 --- a/pkg/bloombuild/protos/compat.go +++ b/pkg/bloombuild/protos/compat.go @@ -7,14 +7,16 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/grafana/loki/v3/pkg/logproto" v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper" "github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb" ) -type GapWithBlocks struct { +type Gap struct { Bounds v1.FingerprintBounds + Series []*v1.Series Blocks []bloomshipper.BlockRef } @@ -25,7 +27,7 @@ type Task struct { Tenant string OwnershipBounds v1.FingerprintBounds TSDB tsdb.SingleTenantTSDBIdentifier - Gaps []GapWithBlocks + Gaps []Gap } func NewTask( @@ -33,10 +35,10 @@ func NewTask( tenant string, bounds v1.FingerprintBounds, tsdb tsdb.SingleTenantTSDBIdentifier, - gaps []GapWithBlocks, + gaps []Gap, ) *Task { return &Task{ - ID: fmt.Sprintf("%s-%s-%s-%d-%d", table.Addr(), tenant, bounds.String(), tsdb.Checksum, len(gaps)), + ID: fmt.Sprintf("%s-%s-%s-%d", table.Addr(), tenant, bounds.String(), len(gaps)), Table: table, Tenant: tenant, @@ -56,12 +58,25 @@ func FromProtoTask(task *ProtoTask) (*Task, error) { return nil, fmt.Errorf("failed to parse tsdb path %s", task.Tsdb) } - gaps := make([]GapWithBlocks, 0, len(task.Gaps)) + gaps := make([]Gap, 0, len(task.Gaps)) for _, gap := range task.Gaps { bounds := v1.FingerprintBounds{ Min: gap.Bounds.Min, Max: gap.Bounds.Max, } + + series := make([]*v1.Series, 0, len(gap.Series)) + for _, s := range gap.Series { + chunks := make(v1.ChunkRefs, 0, len(s.Chunks)) + for _, c := range s.Chunks { + chunks = append(chunks, v1.ChunkRef(*c)) + } + series = append(series, &v1.Series{ + Fingerprint: model.Fingerprint(s.Fingerprint), + Chunks: chunks, + }) + } + blocks := make([]bloomshipper.BlockRef, 0, len(gap.BlockRef)) for _, block := range gap.BlockRef { b, err := bloomshipper.BlockRefFromKey(block) @@ -71,8 +86,9 @@ func FromProtoTask(task *ProtoTask) (*Task, error) { blocks = append(blocks, b) } - gaps = append(gaps, GapWithBlocks{ + gaps = append(gaps, Gap{ Bounds: bounds, + Series: series, Blocks: blocks, }) } @@ -102,11 +118,26 @@ func (t *Task) ToProtoTask() *ProtoTask { blockRefs = append(blockRefs, block.String()) } + series := make([]*ProtoSeries, 0, len(gap.Series)) + for _, s := range gap.Series { + chunks := make([]*logproto.ShortRef, 0, len(s.Chunks)) + for _, c := range s.Chunks { + chunk := logproto.ShortRef(c) + chunks = append(chunks, &chunk) + } + + series = append(series, &ProtoSeries{ + Fingerprint: uint64(s.Fingerprint), + Chunks: chunks, + }) + } + protoGaps = append(protoGaps, &ProtoGapWithBlocks{ Bounds: ProtoFingerprintBounds{ Min: gap.Bounds.Min, Max: gap.Bounds.Max, }, + Series: series, BlockRef: blockRefs, }) } diff --git a/pkg/bloombuild/protos/types.pb.go b/pkg/bloombuild/protos/types.pb.go index e528aa61e9178..f355b64711168 100644 --- a/pkg/bloombuild/protos/types.pb.go +++ b/pkg/bloombuild/protos/types.pb.go @@ -7,6 +7,7 @@ import ( fmt "fmt" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" + logproto "github.com/grafana/loki/v3/pkg/logproto" github_com_prometheus_common_model "github.com/prometheus/common/model" io "io" math "math" @@ -131,15 +132,67 @@ func (m *DayTable) GetPrefix() string { return "" } +type ProtoSeries struct { + Fingerprint uint64 `protobuf:"varint,1,opt,name=fingerprint,proto3" json:"fingerprint,omitempty"` + Chunks []*logproto.ShortRef `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks,omitempty"` +} + +func (m *ProtoSeries) Reset() { *m = ProtoSeries{} } +func (*ProtoSeries) ProtoMessage() {} +func (*ProtoSeries) Descriptor() ([]byte, []int) { + return fileDescriptor_5325fb0610e1e9ae, []int{2} +} +func (m *ProtoSeries) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ProtoSeries) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ProtoSeries.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ProtoSeries) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProtoSeries.Merge(m, src) +} +func (m *ProtoSeries) XXX_Size() int { + return m.Size() +} +func (m *ProtoSeries) XXX_DiscardUnknown() { + xxx_messageInfo_ProtoSeries.DiscardUnknown(m) +} + +var xxx_messageInfo_ProtoSeries proto.InternalMessageInfo + +func (m *ProtoSeries) GetFingerprint() uint64 { + if m != nil { + return m.Fingerprint + } + return 0 +} + +func (m *ProtoSeries) GetChunks() []*logproto.ShortRef { + if m != nil { + return m.Chunks + } + return nil +} + type ProtoGapWithBlocks struct { Bounds ProtoFingerprintBounds `protobuf:"bytes,1,opt,name=bounds,proto3" json:"bounds"` - BlockRef []string `protobuf:"bytes,2,rep,name=blockRef,proto3" json:"blockRef,omitempty"` + Series []*ProtoSeries `protobuf:"bytes,2,rep,name=series,proto3" json:"series,omitempty"` + BlockRef []string `protobuf:"bytes,3,rep,name=blockRef,proto3" json:"blockRef,omitempty"` } func (m *ProtoGapWithBlocks) Reset() { *m = ProtoGapWithBlocks{} } func (*ProtoGapWithBlocks) ProtoMessage() {} func (*ProtoGapWithBlocks) Descriptor() ([]byte, []int) { - return fileDescriptor_5325fb0610e1e9ae, []int{2} + return fileDescriptor_5325fb0610e1e9ae, []int{3} } func (m *ProtoGapWithBlocks) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -175,6 +228,13 @@ func (m *ProtoGapWithBlocks) GetBounds() ProtoFingerprintBounds { return ProtoFingerprintBounds{} } +func (m *ProtoGapWithBlocks) GetSeries() []*ProtoSeries { + if m != nil { + return m.Series + } + return nil +} + func (m *ProtoGapWithBlocks) GetBlockRef() []string { if m != nil { return m.BlockRef @@ -197,7 +257,7 @@ type ProtoTask struct { func (m *ProtoTask) Reset() { *m = ProtoTask{} } func (*ProtoTask) ProtoMessage() {} func (*ProtoTask) Descriptor() ([]byte, []int) { - return fileDescriptor_5325fb0610e1e9ae, []int{3} + return fileDescriptor_5325fb0610e1e9ae, []int{4} } func (m *ProtoTask) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -277,7 +337,7 @@ type ProtoMeta struct { func (m *ProtoMeta) Reset() { *m = ProtoMeta{} } func (*ProtoMeta) ProtoMessage() {} func (*ProtoMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_5325fb0610e1e9ae, []int{4} + return fileDescriptor_5325fb0610e1e9ae, []int{5} } func (m *ProtoMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -336,7 +396,7 @@ type ProtoTaskResult struct { func (m *ProtoTaskResult) Reset() { *m = ProtoTaskResult{} } func (*ProtoTaskResult) ProtoMessage() {} func (*ProtoTaskResult) Descriptor() ([]byte, []int) { - return fileDescriptor_5325fb0610e1e9ae, []int{5} + return fileDescriptor_5325fb0610e1e9ae, []int{6} } func (m *ProtoTaskResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -389,6 +449,7 @@ func (m *ProtoTaskResult) GetCreatedMetas() []*ProtoMeta { func init() { proto.RegisterType((*ProtoFingerprintBounds)(nil), "protos.ProtoFingerprintBounds") proto.RegisterType((*DayTable)(nil), "protos.DayTable") + proto.RegisterType((*ProtoSeries)(nil), "protos.ProtoSeries") proto.RegisterType((*ProtoGapWithBlocks)(nil), "protos.ProtoGapWithBlocks") proto.RegisterType((*ProtoTask)(nil), "protos.ProtoTask") proto.RegisterType((*ProtoMeta)(nil), "protos.ProtoMeta") @@ -398,42 +459,47 @@ func init() { func init() { proto.RegisterFile("pkg/bloombuild/protos/types.proto", fileDescriptor_5325fb0610e1e9ae) } var fileDescriptor_5325fb0610e1e9ae = []byte{ - // 551 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0xb1, 0x6f, 0xd3, 0x4e, - 0x18, 0xb5, 0xe3, 0x34, 0xbf, 0xe6, 0xd2, 0x5f, 0x81, 0x53, 0x55, 0x59, 0x11, 0xba, 0x04, 0x0f, - 0x28, 0x93, 0x2d, 0x05, 0x75, 0x40, 0x62, 0xb2, 0xa2, 0x22, 0x40, 0x95, 0xd0, 0x35, 0x12, 0x12, - 0xdb, 0x39, 0xbe, 0x3a, 0x56, 0x6c, 0x9f, 0xe5, 0x3b, 0xa3, 0x64, 0xe3, 0x4f, 0xe0, 0xcf, 0x60, - 0xe6, 0xaf, 0xe8, 0x98, 0xb1, 0x53, 0x44, 0x9c, 0x05, 0x75, 0xea, 0xc4, 0xc0, 0x84, 0xee, 0xce, - 0x29, 0x09, 0x62, 0x82, 0xe9, 0xbe, 0xf7, 0xdd, 0x77, 0xef, 0x7b, 0xef, 0xc9, 0x06, 0x4f, 0xf2, - 0x59, 0xe4, 0x05, 0x09, 0x63, 0x69, 0x50, 0xc6, 0x49, 0xe8, 0xe5, 0x05, 0x13, 0x8c, 0x7b, 0x62, - 0x91, 0x53, 0xee, 0x2a, 0x00, 0x5b, 0xba, 0xd7, 0x3d, 0x89, 0x58, 0xc4, 0x54, 0xed, 0xc9, 0x4a, - 0xdf, 0x3a, 0x5f, 0x4c, 0x70, 0xfa, 0x56, 0x56, 0xe7, 0x71, 0x16, 0xd1, 0x22, 0x2f, 0xe2, 0x4c, - 0xf8, 0xac, 0xcc, 0x42, 0x0e, 0xdf, 0x00, 0x2b, 0x8d, 0x33, 0xdb, 0xec, 0x9b, 0x83, 0xa6, 0xff, - 0xfc, 0x76, 0xd5, 0x93, 0xf0, 0xc7, 0xaa, 0xe7, 0x46, 0xb1, 0x98, 0x96, 0x81, 0x3b, 0x61, 0xa9, - 0xdc, 0x97, 0x52, 0x31, 0xa5, 0x25, 0xf7, 0x26, 0x2c, 0x4d, 0x59, 0xe6, 0xa5, 0x2c, 0xa4, 0x89, - 0xbb, 0xc3, 0x86, 0xe5, 0x33, 0x45, 0x46, 0xe6, 0x76, 0x63, 0x87, 0x8c, 0xcc, 0xff, 0x8a, 0x8c, - 0xcc, 0x9d, 0xd7, 0xe0, 0x70, 0x44, 0x16, 0x63, 0x12, 0x24, 0x14, 0x3e, 0x05, 0xc7, 0x21, 0x59, - 0x8c, 0xe3, 0x94, 0x72, 0x41, 0xd2, 0xfc, 0xe2, 0x52, 0x09, 0xb6, 0xf0, 0x6f, 0x5d, 0x78, 0x0a, - 0x5a, 0x79, 0x41, 0xaf, 0x62, 0xad, 0xa1, 0x8d, 0x6b, 0xe4, 0xcc, 0x01, 0x54, 0xfe, 0x5f, 0x92, - 0xfc, 0x5d, 0x2c, 0xa6, 0x7e, 0xc2, 0x26, 0x33, 0x0e, 0xcf, 0x41, 0x2b, 0x50, 0x29, 0x28, 0xb6, - 0xce, 0x10, 0xe9, 0xb8, 0xb8, 0xfb, 0xe7, 0xac, 0xfc, 0xe3, 0xeb, 0x55, 0xcf, 0xb8, 0x5d, 0xf5, - 0xea, 0x57, 0xb8, 0x3e, 0x61, 0x17, 0x1c, 0x06, 0x92, 0x11, 0xd3, 0x2b, 0xbb, 0xd1, 0xb7, 0x06, - 0x6d, 0x7c, 0x8f, 0x9d, 0xef, 0x26, 0x68, 0x2b, 0xba, 0x31, 0xe1, 0x33, 0x78, 0x0c, 0x1a, 0x71, - 0xa8, 0xb6, 0xb5, 0x71, 0x23, 0x0e, 0xe1, 0x19, 0x38, 0x10, 0xd2, 0xa0, 0x92, 0xdb, 0x19, 0x3e, - 0xdc, 0x0a, 0xd8, 0x1a, 0xf7, 0xff, 0xaf, 0x57, 0xea, 0x31, 0xac, 0x0f, 0x69, 0x53, 0xd0, 0x8c, - 0x64, 0xc2, 0xb6, 0xb4, 0x4d, 0x8d, 0x76, 0x0c, 0x35, 0xff, 0xc9, 0x10, 0x04, 0x4d, 0xc1, 0xc3, - 0xc0, 0x3e, 0x50, 0xec, 0xaa, 0x86, 0x2e, 0x68, 0x46, 0x24, 0xe7, 0x76, 0xab, 0x6f, 0x0d, 0x3a, - 0xc3, 0xee, 0x1e, 0xf3, 0x5e, 0xac, 0x58, 0xcd, 0x39, 0x51, 0xed, 0xfb, 0x82, 0x0a, 0x02, 0x6d, - 0xf0, 0x5f, 0x4a, 0x05, 0x91, 0x01, 0x69, 0xf3, 0x5b, 0x08, 0x1d, 0x70, 0xc4, 0x59, 0x59, 0x4c, - 0x28, 0x1f, 0x5f, 0x8e, 0x7c, 0x5e, 0xe7, 0xb7, 0xd7, 0x83, 0x8f, 0x41, 0x7b, 0x9b, 0x27, 0xb7, - 0x2d, 0x35, 0xf0, 0xab, 0xe1, 0x7c, 0x00, 0x0f, 0xee, 0x03, 0xc6, 0x94, 0x97, 0x89, 0x50, 0xf9, - 0x10, 0x3e, 0x7b, 0x35, 0xaa, 0xb7, 0xd5, 0x08, 0x9e, 0x80, 0x03, 0x5a, 0x14, 0xac, 0xa8, 0xbf, - 0x0e, 0x0d, 0xe0, 0x19, 0x38, 0x9a, 0x14, 0x94, 0x08, 0x1a, 0x4a, 0xad, 0x7a, 0x43, 0x67, 0xf8, - 0x68, 0xcf, 0xa1, 0xbc, 0xc1, 0x7b, 0x63, 0xfe, 0x8b, 0xe5, 0x1a, 0x19, 0x37, 0x6b, 0x64, 0xdc, - 0xad, 0x91, 0xf9, 0xb1, 0x42, 0xe6, 0xe7, 0x0a, 0x99, 0xd7, 0x15, 0x32, 0x97, 0x15, 0x32, 0xbf, - 0x56, 0xc8, 0xfc, 0x56, 0x21, 0xe3, 0xae, 0x42, 0xe6, 0xa7, 0x0d, 0x32, 0x96, 0x1b, 0x64, 0xdc, - 0x6c, 0x90, 0xf1, 0xbe, 0xfe, 0x51, 0x03, 0x7d, 0x3e, 0xfb, 0x19, 0x00, 0x00, 0xff, 0xff, 0xdc, - 0x0e, 0x2e, 0xd1, 0xdc, 0x03, 0x00, 0x00, + // 630 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0x3f, 0x6f, 0xd3, 0x40, + 0x1c, 0xb5, 0x93, 0x34, 0x34, 0x97, 0x52, 0xe0, 0xa8, 0x2a, 0x2b, 0x42, 0x97, 0xe0, 0x01, 0x55, + 0x20, 0x39, 0x52, 0x50, 0x07, 0x24, 0x26, 0xab, 0x2a, 0x02, 0x54, 0x09, 0x5d, 0x22, 0x21, 0xc1, + 0x74, 0x8e, 0x2f, 0x8e, 0x15, 0xdb, 0x67, 0xf9, 0xce, 0xd0, 0x6c, 0x7c, 0x04, 0xbe, 0x04, 0x12, + 0x33, 0x9f, 0xa2, 0x63, 0xc7, 0x4e, 0x11, 0x75, 0x17, 0xd4, 0xa9, 0x13, 0x03, 0x13, 0xba, 0x3f, + 0x69, 0x13, 0xc4, 0x04, 0xd3, 0xdd, 0xfb, 0xdd, 0xef, 0xde, 0xef, 0xbd, 0x77, 0x96, 0xc1, 0xc3, + 0x7c, 0x16, 0xf5, 0x83, 0x84, 0xb1, 0x34, 0x28, 0xe3, 0x24, 0xec, 0xe7, 0x05, 0x13, 0x8c, 0xf7, + 0xc5, 0x3c, 0xa7, 0xdc, 0x53, 0x00, 0x36, 0x75, 0xad, 0xb3, 0x13, 0xb1, 0x88, 0xa9, 0x7d, 0x5f, + 0xee, 0xf4, 0x69, 0xa7, 0x2b, 0x09, 0x12, 0x16, 0xe9, 0x03, 0xc5, 0x14, 0x11, 0x41, 0x3f, 0x92, + 0xb9, 0x6e, 0x70, 0xbf, 0xd9, 0x60, 0xf7, 0x8d, 0xdc, 0x1d, 0xc6, 0x59, 0x44, 0x8b, 0xbc, 0x88, + 0x33, 0xe1, 0xb3, 0x32, 0x0b, 0x39, 0x7c, 0x0d, 0xea, 0x69, 0x9c, 0x39, 0x76, 0xcf, 0xde, 0x6b, + 0xf8, 0xcf, 0x2e, 0x17, 0x5d, 0x09, 0x7f, 0x2d, 0xba, 0x5e, 0x14, 0x8b, 0x69, 0x19, 0x78, 0x63, + 0x96, 0x4a, 0x41, 0x29, 0x15, 0x53, 0x5a, 0xf2, 0xfe, 0x98, 0xa5, 0x29, 0xcb, 0xfa, 0x29, 0x0b, + 0x69, 0xe2, 0xad, 0xb0, 0x61, 0x79, 0x4d, 0x91, 0x91, 0x63, 0xa7, 0xb6, 0x42, 0x46, 0x8e, 0xff, + 0x89, 0x8c, 0x1c, 0xbb, 0xaf, 0xc0, 0xe6, 0x01, 0x99, 0x8f, 0x48, 0x90, 0x50, 0xf8, 0x08, 0x6c, + 0x87, 0x64, 0x3e, 0x8a, 0x53, 0xca, 0x05, 0x49, 0xf3, 0xa3, 0xa1, 0x12, 0x5c, 0xc7, 0x7f, 0x54, + 0xe1, 0x2e, 0x68, 0xe6, 0x05, 0x9d, 0xc4, 0x5a, 0x43, 0x0b, 0x1b, 0xe4, 0xbe, 0x07, 0x6d, 0xe5, + 0x7f, 0x48, 0x8b, 0x98, 0x72, 0xd8, 0x03, 0xed, 0xc9, 0xcd, 0x38, 0x6d, 0x1e, 0xaf, 0x96, 0xe0, + 0x63, 0xd0, 0x1c, 0x4f, 0xcb, 0x6c, 0xc6, 0x9d, 0x5a, 0xaf, 0xbe, 0xd7, 0x1e, 0x40, 0x6f, 0x99, + 0xaf, 0x37, 0x9c, 0xb2, 0x42, 0x60, 0x3a, 0xc1, 0xa6, 0xc3, 0xfd, 0x62, 0x03, 0xa8, 0xd8, 0x5f, + 0x90, 0xfc, 0x6d, 0x2c, 0xa6, 0x7e, 0xc2, 0xc6, 0x33, 0x0e, 0x0f, 0x41, 0x33, 0x50, 0x19, 0x2b, + 0xfe, 0xf6, 0x00, 0xe9, 0xc7, 0xe0, 0xde, 0xdf, 0x5f, 0xc2, 0xdf, 0x3e, 0x59, 0x74, 0xad, 0xcb, + 0x45, 0xd7, 0xdc, 0xc2, 0x66, 0x85, 0x4f, 0x40, 0x93, 0x2b, 0xd9, 0x46, 0xca, 0xfd, 0x35, 0x1e, + 0xed, 0x08, 0x9b, 0x16, 0xd8, 0x01, 0x9b, 0x81, 0x1c, 0x8f, 0xe9, 0xc4, 0xa9, 0xf7, 0xea, 0x7b, + 0x2d, 0x7c, 0x8d, 0xdd, 0x9f, 0x36, 0x68, 0xa9, 0x3b, 0x23, 0xc2, 0x67, 0x70, 0x1b, 0xd4, 0xe2, + 0x50, 0x49, 0x6b, 0xe1, 0x5a, 0x1c, 0xc2, 0x7d, 0xb0, 0x21, 0x64, 0xd6, 0x2a, 0xb9, 0xf6, 0xe0, + 0xee, 0x72, 0xca, 0xf2, 0x0d, 0xfc, 0xdb, 0x46, 0x9f, 0x6e, 0xc3, 0x7a, 0x91, 0x89, 0x0b, 0x9a, + 0x91, 0x4c, 0x38, 0x75, 0x9d, 0xb8, 0x46, 0x2b, 0xee, 0x1b, 0xff, 0xe5, 0x1e, 0x82, 0x86, 0xe0, + 0x61, 0xe0, 0x6c, 0x28, 0x76, 0xb5, 0x87, 0x1e, 0x68, 0x44, 0x24, 0xe7, 0x4e, 0x53, 0xe5, 0xd1, + 0x59, 0x63, 0x5e, 0x7b, 0x03, 0xac, 0xfa, 0xdc, 0xc8, 0xf8, 0x3e, 0xa2, 0x82, 0x40, 0x07, 0xdc, + 0x4a, 0xa9, 0x20, 0x32, 0x20, 0x6d, 0x7e, 0x09, 0xa1, 0x0b, 0xb6, 0x38, 0x2b, 0x8b, 0x31, 0xe5, + 0xa3, 0xe1, 0x81, 0xaf, 0xe3, 0x6e, 0xe1, 0xb5, 0x1a, 0x7c, 0x00, 0x5a, 0xcb, 0x3c, 0xb9, 0x09, + 0xf8, 0xa6, 0xe0, 0x7e, 0x00, 0x77, 0xae, 0x03, 0xc6, 0x94, 0x97, 0x89, 0x50, 0xf9, 0x10, 0x3e, + 0x7b, 0x79, 0x60, 0xa6, 0x19, 0x04, 0x77, 0xc0, 0x06, 0x2d, 0x0a, 0x56, 0x98, 0x0f, 0x55, 0x03, + 0xb8, 0x0f, 0xb6, 0xc6, 0x05, 0x25, 0x82, 0x86, 0x52, 0xab, 0x9e, 0xd0, 0x1e, 0xdc, 0x5b, 0x73, + 0x28, 0x4f, 0xf0, 0x5a, 0x9b, 0xff, 0xfc, 0xf4, 0x1c, 0x59, 0x67, 0xe7, 0xc8, 0xba, 0x3a, 0x47, + 0xf6, 0xa7, 0x0a, 0xd9, 0x5f, 0x2b, 0x64, 0x9f, 0x54, 0xc8, 0x3e, 0xad, 0x90, 0xfd, 0xbd, 0x42, + 0xf6, 0x8f, 0x0a, 0x59, 0x57, 0x15, 0xb2, 0x3f, 0x5f, 0x20, 0xeb, 0xf4, 0x02, 0x59, 0x67, 0x17, + 0xc8, 0x7a, 0x67, 0x7e, 0x2a, 0x81, 0x5e, 0x9f, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0xf9, 0x38, + 0x02, 0xe1, 0x88, 0x04, 0x00, 0x00, } func (this *ProtoFingerprintBounds) Equal(that interface{}) bool { @@ -490,6 +556,38 @@ func (this *DayTable) Equal(that interface{}) bool { } return true } +func (this *ProtoSeries) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ProtoSeries) + if !ok { + that2, ok := that.(ProtoSeries) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Fingerprint != that1.Fingerprint { + return false + } + if len(this.Chunks) != len(that1.Chunks) { + return false + } + for i := range this.Chunks { + if !this.Chunks[i].Equal(that1.Chunks[i]) { + return false + } + } + return true +} func (this *ProtoGapWithBlocks) Equal(that interface{}) bool { if that == nil { return this == nil @@ -512,6 +610,14 @@ func (this *ProtoGapWithBlocks) Equal(that interface{}) bool { if !this.Bounds.Equal(&that1.Bounds) { return false } + if len(this.Series) != len(that1.Series) { + return false + } + for i := range this.Series { + if !this.Series[i].Equal(that1.Series[i]) { + return false + } + } if len(this.BlockRef) != len(that1.BlockRef) { return false } @@ -663,13 +769,29 @@ func (this *DayTable) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func (this *ProtoGapWithBlocks) GoString() string { +func (this *ProtoSeries) GoString() string { if this == nil { return "nil" } s := make([]string, 0, 6) + s = append(s, "&protos.ProtoSeries{") + s = append(s, "Fingerprint: "+fmt.Sprintf("%#v", this.Fingerprint)+",\n") + if this.Chunks != nil { + s = append(s, "Chunks: "+fmt.Sprintf("%#v", this.Chunks)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func (this *ProtoGapWithBlocks) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 7) s = append(s, "&protos.ProtoGapWithBlocks{") s = append(s, "Bounds: "+strings.Replace(this.Bounds.GoString(), `&`, ``, 1)+",\n") + if this.Series != nil { + s = append(s, "Series: "+fmt.Sprintf("%#v", this.Series)+",\n") + } s = append(s, "BlockRef: "+fmt.Sprintf("%#v", this.BlockRef)+",\n") s = append(s, "}") return strings.Join(s, "") @@ -793,6 +915,48 @@ func (m *DayTable) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ProtoSeries) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProtoSeries) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ProtoSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Chunks) > 0 { + for iNdEx := len(m.Chunks) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Chunks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if m.Fingerprint != 0 { + i = encodeVarintTypes(dAtA, i, uint64(m.Fingerprint)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *ProtoGapWithBlocks) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -819,6 +983,20 @@ func (m *ProtoGapWithBlocks) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.BlockRef[iNdEx]) i = encodeVarintTypes(dAtA, i, uint64(len(m.BlockRef[iNdEx]))) i-- + dAtA[i] = 0x1a + } + } + if len(m.Series) > 0 { + for iNdEx := len(m.Series) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Series[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- dAtA[i] = 0x12 } } @@ -1054,6 +1232,24 @@ func (m *DayTable) Size() (n int) { return n } +func (m *ProtoSeries) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Fingerprint != 0 { + n += 1 + sovTypes(uint64(m.Fingerprint)) + } + if len(m.Chunks) > 0 { + for _, e := range m.Chunks { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } + return n +} + func (m *ProtoGapWithBlocks) Size() (n int) { if m == nil { return 0 @@ -1062,6 +1258,12 @@ func (m *ProtoGapWithBlocks) Size() (n int) { _ = l l = m.Bounds.Size() n += 1 + l + sovTypes(uint64(l)) + if len(m.Series) > 0 { + for _, e := range m.Series { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } if len(m.BlockRef) > 0 { for _, s := range m.BlockRef { l = len(s) @@ -1178,12 +1380,34 @@ func (this *DayTable) String() string { }, "") return s } +func (this *ProtoSeries) String() string { + if this == nil { + return "nil" + } + repeatedStringForChunks := "[]*ShortRef{" + for _, f := range this.Chunks { + repeatedStringForChunks += strings.Replace(fmt.Sprintf("%v", f), "ShortRef", "logproto.ShortRef", 1) + "," + } + repeatedStringForChunks += "}" + s := strings.Join([]string{`&ProtoSeries{`, + `Fingerprint:` + fmt.Sprintf("%v", this.Fingerprint) + `,`, + `Chunks:` + repeatedStringForChunks + `,`, + `}`, + }, "") + return s +} func (this *ProtoGapWithBlocks) String() string { if this == nil { return "nil" } + repeatedStringForSeries := "[]*ProtoSeries{" + for _, f := range this.Series { + repeatedStringForSeries += strings.Replace(f.String(), "ProtoSeries", "ProtoSeries", 1) + "," + } + repeatedStringForSeries += "}" s := strings.Join([]string{`&ProtoGapWithBlocks{`, `Bounds:` + strings.Replace(strings.Replace(this.Bounds.String(), "ProtoFingerprintBounds", "ProtoFingerprintBounds", 1), `&`, ``, 1) + `,`, + `Series:` + repeatedStringForSeries + `,`, `BlockRef:` + fmt.Sprintf("%v", this.BlockRef) + `,`, `}`, }, "") @@ -1441,6 +1665,112 @@ func (m *DayTable) Unmarshal(dAtA []byte) error { } return nil } +func (m *ProtoSeries) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProtoSeries: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProtoSeries: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Fingerprint", wireType) + } + m.Fingerprint = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Fingerprint |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Chunks", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Chunks = append(m.Chunks, &logproto.ShortRef{}) + if err := m.Chunks[len(m.Chunks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ProtoGapWithBlocks) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1504,6 +1834,40 @@ func (m *ProtoGapWithBlocks) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Series", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Series = append(m.Series, &ProtoSeries{}) + if err := m.Series[len(m.Series)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field BlockRef", wireType) } diff --git a/pkg/bloombuild/protos/types.proto b/pkg/bloombuild/protos/types.proto index 55ae89625abe6..9e63dd1adb604 100644 --- a/pkg/bloombuild/protos/types.proto +++ b/pkg/bloombuild/protos/types.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package protos; import "gogoproto/gogo.proto"; +import "pkg/logproto/bloomgateway.proto"; option go_package = "protos"; option (gogoproto.marshaler_all) = true; @@ -27,12 +28,18 @@ message DayTable { string prefix = 2; } +message ProtoSeries { + uint64 fingerprint = 1; + repeated logproto.ShortRef chunks = 2; +} + message ProtoGapWithBlocks { ProtoFingerprintBounds bounds = 1 [ (gogoproto.nullable) = false, (gogoproto.jsontag) = "bounds" ]; - repeated string blockRef = 2; + repeated ProtoSeries series = 2; + repeated string blockRef = 3; } // TODO: Define BlockRef and SingleTenantTSDBIdentifier as messages so we can use them right away diff --git a/pkg/chunkenc/memchunk.go b/pkg/chunkenc/memchunk.go index f6197888a854f..328e91c94deb3 100644 --- a/pkg/chunkenc/memchunk.go +++ b/pkg/chunkenc/memchunk.go @@ -441,13 +441,20 @@ func newByteChunk(b []byte, blockSize, targetSize int, fromCheckpoint bool) (*Me metasOffset := uint64(0) metasLen := uint64(0) + // There is a rare issue where chunks built by Loki have incorrect offset for some blocks which causes Loki to fail to read those chunks. + // While the root cause is yet to be identified, we will try to read those problematic chunks using the expected offset for blocks calculated using other relative offsets in the chunk. + expectedBlockOffset := 0 if version >= ChunkFormatV4 { - // version >= 4 starts writing length of sections after their offsets + // version >= 4 starts writing length of sections before their offsets metasLen, metasOffset = readSectionLenAndOffset(chunkMetasSectionIdx) + structuredMetadataLength, structuredMetadataOffset := readSectionLenAndOffset(chunkStructuredMetadataSectionIdx) + expectedBlockOffset = int(structuredMetadataLength + structuredMetadataOffset + 4) } else { // version <= 3 does not store length of metas. metas are followed by metasOffset + hash and then the chunk ends metasOffset = binary.BigEndian.Uint64(b[len(b)-8:]) metasLen = uint64(len(b)-(8+4)) - metasOffset + // version 1 writes blocks after version number while version 2 and 3 write blocks after chunk encoding + expectedBlockOffset = len(b) - len(db.b) } mb := b[metasOffset : metasOffset+metasLen] db = decbuf{b: mb} @@ -476,15 +483,35 @@ func newByteChunk(b []byte, blockSize, targetSize int, fromCheckpoint bool) (*Me blk.uncompressedSize = db.uvarint() } l := db.uvarint() - blk.b = b[blk.offset : blk.offset+l] - // Verify checksums. - expCRC := binary.BigEndian.Uint32(b[blk.offset+l:]) - if expCRC != crc32.Checksum(blk.b, castagnoliTable) { - _ = level.Error(util_log.Logger).Log("msg", "Checksum does not match for a block in chunk, this block will be skipped", "err", ErrInvalidChecksum) - continue + invalidBlockErr := validateBlock(b, blk.offset, l) + if invalidBlockErr != nil { + level.Error(util_log.Logger).Log("msg", "invalid block found", "err", invalidBlockErr) + // if block is expected to have different offset than what is encoded, see if we get a valid block using expected offset + if blk.offset != expectedBlockOffset { + _ = level.Error(util_log.Logger).Log("msg", "block offset does not match expected one, will try reading with expected offset", "actual", blk.offset, "expected", expectedBlockOffset) + blk.offset = expectedBlockOffset + if err := validateBlock(b, blk.offset, l); err != nil { + level.Error(util_log.Logger).Log("msg", "could not find valid block using expected offset", "err", err) + } else { + invalidBlockErr = nil + level.Info(util_log.Logger).Log("msg", "valid block found using expected offset") + } + } + + // if the block read with expected offset is still invalid, do not continue further + if invalidBlockErr != nil { + if errors.Is(invalidBlockErr, ErrInvalidChecksum) { + expectedBlockOffset += l + 4 + continue + } + return nil, invalidBlockErr + } } + // next block starts at current block start + current block length + checksum + expectedBlockOffset = blk.offset + l + 4 + blk.b = b[blk.offset : blk.offset+l] bc.blocks = append(bc.blocks, blk) // Update the counter used to track the size of cut blocks. @@ -1693,3 +1720,21 @@ func (e *sampleBufferedIterator) StreamHash() uint64 { return e.extractor.BaseLa func (e *sampleBufferedIterator) At() logproto.Sample { return e.cur } + +// validateBlock validates block by doing following checks: +// 1. Offset+length do not overrun size of the chunk from which we are reading the block. +// 2. Checksum of the block we will read matches the stored checksum in the chunk. +func validateBlock(chunkBytes []byte, offset, length int) error { + if offset+length > len(chunkBytes) { + return fmt.Errorf("offset %d + length %d exceeds chunk length %d", offset, length, len(chunkBytes)) + } + + blockBytes := chunkBytes[offset : offset+length] + // Verify checksums. + expCRC := binary.BigEndian.Uint32(chunkBytes[offset+length:]) + if expCRC != crc32.Checksum(blockBytes, castagnoliTable) { + return ErrInvalidChecksum + } + + return nil +} diff --git a/pkg/chunkenc/memchunk_test.go b/pkg/chunkenc/memchunk_test.go index 6c48a28b0650f..daa97a2616917 100644 --- a/pkg/chunkenc/memchunk_test.go +++ b/pkg/chunkenc/memchunk_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/binary" "fmt" + "hash" "math" "math/rand" "sort" @@ -2044,3 +2045,119 @@ func TestMemChunk_IteratorWithStructuredMetadata(t *testing.T) { }) } } + +func TestDecodeChunkIncorrectBlockOffset(t *testing.T) { + // use small block size to build multiple blocks in the test chunk + blockSize := 10 + + for _, format := range allPossibleFormats { + t.Run(fmt.Sprintf("chunkFormat:%v headBlockFmt:%v", format.chunkFormat, format.headBlockFmt), func(t *testing.T) { + for incorrectOffsetBlockNum := 0; incorrectOffsetBlockNum < 3; incorrectOffsetBlockNum++ { + t.Run(fmt.Sprintf("inorrect offset block: %d", incorrectOffsetBlockNum), func(t *testing.T) { + chk := NewMemChunk(format.chunkFormat, EncNone, format.headBlockFmt, blockSize, testTargetSize) + ts := time.Now().Unix() + for i := 0; i < 3; i++ { + dup, err := chk.Append(&logproto.Entry{ + Timestamp: time.Now(), + Line: fmt.Sprintf("%d-%d", ts, i), + StructuredMetadata: []logproto.LabelAdapter{ + {Name: "foo", Value: fmt.Sprintf("%d-%d", ts, i)}, + }, + }) + require.NoError(t, err) + require.False(t, dup) + } + + require.Len(t, chk.blocks, 3) + + b, err := chk.Bytes() + require.NoError(t, err) + + metasOffset := binary.BigEndian.Uint64(b[len(b)-8:]) + + w := bytes.NewBuffer(nil) + eb := EncodeBufferPool.Get().(*encbuf) + defer EncodeBufferPool.Put(eb) + + crc32Hash := crc32HashPool.Get().(hash.Hash32) + defer crc32HashPool.Put(crc32Hash) + + crc32Hash.Reset() + eb.reset() + + // BEGIN - code copied from writeTo func starting from encoding of block metas to change offset of a block + eb.putUvarint(len(chk.blocks)) + + for i, b := range chk.blocks { + eb.putUvarint(b.numEntries) + eb.putVarint64(b.mint) + eb.putVarint64(b.maxt) + // change offset of one block + blockOffset := b.offset + if i == incorrectOffsetBlockNum { + blockOffset += 5 + } + eb.putUvarint(blockOffset) + if chk.format >= ChunkFormatV3 { + eb.putUvarint(b.uncompressedSize) + } + eb.putUvarint(len(b.b)) + } + metasLen := len(eb.get()) + eb.putHash(crc32Hash) + + _, err = w.Write(eb.get()) + require.NoError(t, err) + + if chk.format >= ChunkFormatV4 { + // Write structured metadata offset and length + eb.reset() + + eb.putBE64int(int(binary.BigEndian.Uint64(b[len(b)-32:]))) + eb.putBE64int(int(binary.BigEndian.Uint64(b[len(b)-24:]))) + _, err = w.Write(eb.get()) + require.NoError(t, err) + } + + // Write the metasOffset. + eb.reset() + if chk.format >= ChunkFormatV4 { + eb.putBE64int(metasLen) + } + eb.putBE64int(int(metasOffset)) + _, err = w.Write(eb.get()) + require.NoError(t, err) + // END - code copied from writeTo func + + // build chunk using pre-block meta section + rewritten remainder of the chunk with incorrect offset for a block + chkWithIncorrectOffset := make([]byte, int(metasOffset)+w.Len()) + copy(chkWithIncorrectOffset, b[:metasOffset]) + copy(chkWithIncorrectOffset[metasOffset:], w.Bytes()) + + // decoding the problematic chunk should succeed + decodedChkWithIncorrectOffset, err := newByteChunk(chkWithIncorrectOffset, blockSize, testTargetSize, false) + require.NoError(t, err) + + require.Len(t, decodedChkWithIncorrectOffset.blocks, len(chk.blocks)) + + // both chunks should have same log lines + origChunkItr, err := chk.Iterator(context.Background(), time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, log.NewNoopPipeline().ForStream(labels.Labels{})) + require.NoError(t, err) + + corruptChunkItr, err := decodedChkWithIncorrectOffset.Iterator(context.Background(), time.Unix(0, 0), time.Unix(0, math.MaxInt64), logproto.FORWARD, log.NewNoopPipeline().ForStream(labels.Labels{})) + require.NoError(t, err) + + numEntriesFound := 0 + for origChunkItr.Next() { + numEntriesFound++ + require.True(t, corruptChunkItr.Next()) + require.Equal(t, origChunkItr.At(), corruptChunkItr.At()) + } + + require.False(t, corruptChunkItr.Next()) + require.Equal(t, 3, numEntriesFound) + }) + } + }) + } +} diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index ebe531e2ab4b2..cd9524a9168ec 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -61,8 +61,6 @@ const ( ringAutoForgetUnhealthyPeriods = 2 - labelServiceName = "service_name" - serviceUnknown = "unknown_service" levelLabel = "detected_level" logLevelDebug = "debug" logLevelInfo = "info" @@ -438,6 +436,10 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log pushSize += len(entry.Line) } stream.Entries = stream.Entries[:n] + if len(stream.Entries) == 0 { + // Empty stream after validating all the entries + continue + } shardStreamsCfg := d.validator.Limits.ShardStreams(tenantID) if shardStreamsCfg.Enabled { @@ -785,20 +787,6 @@ func (d *Distributor) parseStreamLabels(vContext validationContext, key string, return nil, "", 0, err } - // We do not want to count service_name added by us in the stream limit so adding it after validating original labels. - if !ls.Has(labelServiceName) && len(vContext.discoverServiceName) > 0 { - serviceName := serviceUnknown - for _, labelName := range vContext.discoverServiceName { - if labelVal := ls.Get(labelName); labelVal != "" { - serviceName = labelVal - break - } - } - - ls = labels.NewBuilder(ls).Set(labelServiceName, serviceName).Labels() - stream.Labels = ls.String() - } - lsHash := ls.Hash() d.labelCache.Add(key, labelData{ls, lsHash}) diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index 19019e62dd4a3..c21b1e2561cd2 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -103,7 +103,6 @@ func TestDistributor(t *testing.T) { t.Run(fmt.Sprintf("[%d](lines=%v)", i, tc.lines), func(t *testing.T) { limits := &validation.Limits{} flagext.DefaultValues(limits) - limits.DiscoverServiceName = nil limits.IngestionRateMB = ingestionRateLimit limits.IngestionBurstSizeMB = ingestionRateLimit limits.MaxLineSize = fe.ByteSize(tc.maxLineSize) @@ -140,20 +139,17 @@ func TestDistributor(t *testing.T) { func Test_IncrementTimestamp(t *testing.T) { incrementingDisabled := &validation.Limits{} flagext.DefaultValues(incrementingDisabled) - incrementingDisabled.DiscoverServiceName = nil incrementingDisabled.RejectOldSamples = false incrementingDisabled.DiscoverLogLevels = false incrementingEnabled := &validation.Limits{} flagext.DefaultValues(incrementingEnabled) - incrementingEnabled.DiscoverServiceName = nil incrementingEnabled.RejectOldSamples = false incrementingEnabled.IncrementDuplicateTimestamp = true incrementingEnabled.DiscoverLogLevels = false defaultLimits := &validation.Limits{} flagext.DefaultValues(defaultLimits) - now := time.Now() defaultLimits.DiscoverLogLevels = false tests := map[string]struct { @@ -401,34 +397,6 @@ func Test_IncrementTimestamp(t *testing.T) { }, }, }, - "default limit adding service_name label": { - limits: defaultLimits, - push: &logproto.PushRequest{ - Streams: []logproto.Stream{ - { - Labels: "{job=\"foo\"}", - Entries: []logproto.Entry{ - {Timestamp: now.Add(-2 * time.Second), Line: "hey1"}, - {Timestamp: now.Add(-time.Second), Line: "hey2"}, - {Timestamp: now, Line: "hey3"}, - }, - }, - }, - }, - expectedPush: &logproto.PushRequest{ - Streams: []logproto.Stream{ - { - Labels: "{job=\"foo\", service_name=\"foo\"}", - Hash: 0x86ca305b6d86e8b0, - Entries: []logproto.Entry{ - {Timestamp: now.Add(-2 * time.Second), Line: "hey1"}, - {Timestamp: now.Add(-time.Second), Line: "hey2"}, - {Timestamp: now, Line: "hey3"}, - }, - }, - }, - }, - }, } for testName, testData := range tests { @@ -448,7 +416,6 @@ func Test_IncrementTimestamp(t *testing.T) { func TestDistributorPushConcurrently(t *testing.T) { limits := &validation.Limits{} flagext.DefaultValues(limits) - limits.DiscoverServiceName = nil distributors, ingesters := prepare(t, 1, 5, limits, nil) @@ -552,40 +519,46 @@ func Test_SortLabelsOnPush(t *testing.T) { topVal := ingester.Peek() require.Equal(t, `{a="b", buzz="f", service_name="foo"}`, topVal.Streams[0].Labels) }) +} - t.Run("with service_name added during ingestion", func(t *testing.T) { +func Test_TruncateLogLines(t *testing.T) { + setup := func() (*validation.Limits, *mockIngester) { limits := &validation.Limits{} flagext.DefaultValues(limits) - ingester := &mockIngester{} + + limits.MaxLineSize = 5 + limits.MaxLineSizeTruncate = true + return limits, &mockIngester{} + } + + t.Run("it truncates lines to MaxLineSize when MaxLineSizeTruncate is true", func(t *testing.T) { + limits, ingester := setup() distributors, _ := prepare(t, 1, 5, limits, func(addr string) (ring_client.PoolClient, error) { return ingester, nil }) - request := makeWriteRequest(10, 10) - request.Streams[0].Labels = `{buzz="f", x="y", a="b"}` - _, err := distributors[0].Push(ctx, request) + _, err := distributors[0].Push(ctx, makeWriteRequest(1, 10)) require.NoError(t, err) topVal := ingester.Peek() - require.Equal(t, `{a="b", buzz="f", service_name="unknown_service", x="y"}`, topVal.Streams[0].Labels) + require.Len(t, topVal.Streams[0].Entries[0].Line, 5) }) } -func Test_TruncateLogLines(t *testing.T) { +func Test_DiscardEmptyStreamsAfterValidation(t *testing.T) { setup := func() (*validation.Limits, *mockIngester) { limits := &validation.Limits{} flagext.DefaultValues(limits) limits.MaxLineSize = 5 - limits.MaxLineSizeTruncate = true return limits, &mockIngester{} } - t.Run("it truncates lines to MaxLineSize when MaxLineSizeTruncate is true", func(t *testing.T) { + t.Run("it discards invalid entries and discards resulting empty streams completely", func(t *testing.T) { limits, ingester := setup() distributors, _ := prepare(t, 1, 5, limits, func(addr string) (ring_client.PoolClient, error) { return ingester, nil }) _, err := distributors[0].Push(ctx, makeWriteRequest(1, 10)) - require.NoError(t, err) + require.Equal(t, err, httpgrpc.Errorf(http.StatusBadRequest, fmt.Sprintf(validation.LineTooLongErrorMsg, 5, "{foo=\"bar\"}", 10))) topVal := ingester.Peek() - require.Len(t, topVal.Streams[0].Entries[0].Line, 5) + require.Nil(t, topVal) }) } @@ -865,53 +838,9 @@ func TestParseStreamLabels(t *testing.T) { expectedErr error generateLimits func() *validation.Limits }{ - { - name: "service name label mapping disabled", - generateLimits: func() *validation.Limits { - limits := &validation.Limits{} - flagext.DefaultValues(limits) - limits.DiscoverServiceName = nil - return limits - }, - origLabels: `{foo="bar"}`, - expectedLabels: labels.Labels{ - { - Name: "foo", - Value: "bar", - }, - }, - }, - { - name: "no labels defined - service name label mapping disabled", - generateLimits: func() *validation.Limits { - limits := &validation.Limits{} - flagext.DefaultValues(limits) - limits.DiscoverServiceName = nil - return limits - }, - origLabels: `{}`, - expectedErr: fmt.Errorf(validation.MissingLabelsErrorMsg), - }, - { - name: "service name label enabled", - origLabels: `{foo="bar"}`, - generateLimits: func() *validation.Limits { - return defaultLimit - }, - expectedLabels: labels.Labels{ - { - Name: "foo", - Value: "bar", - }, - { - Name: labelServiceName, - Value: serviceUnknown, - }, - }, - }, { name: "service name label should not get counted against max labels count", - origLabels: `{foo="bar"}`, + origLabels: `{foo="bar", service_name="unknown_service"}`, generateLimits: func() *validation.Limits { limits := &validation.Limits{} flagext.DefaultValues(limits) @@ -924,33 +853,8 @@ func TestParseStreamLabels(t *testing.T) { Value: "bar", }, { - Name: labelServiceName, - Value: serviceUnknown, - }, - }, - }, - { - name: "use label service as service name", - origLabels: `{container="nginx", foo="bar", service="auth"}`, - generateLimits: func() *validation.Limits { - return defaultLimit - }, - expectedLabels: labels.Labels{ - { - Name: "container", - Value: "nginx", - }, - { - Name: "foo", - Value: "bar", - }, - { - Name: "service", - Value: "auth", - }, - { - Name: labelServiceName, - Value: "auth", + Name: loghttp_push.LabelServiceName, + Value: loghttp_push.ServiceUnknown, }, }, }, @@ -1542,7 +1446,6 @@ func Test_DetectLogLevels(t *testing.T) { flagext.DefaultValues(limits) limits.DiscoverLogLevels = discoverLogLevels - limits.DiscoverServiceName = nil limits.AllowStructuredMetadata = true return limits, &mockIngester{} } diff --git a/pkg/distributor/validator.go b/pkg/distributor/validator.go index 2ef4c78cff94a..b4f730a58a7fa 100644 --- a/pkg/distributor/validator.go +++ b/pkg/distributor/validator.go @@ -157,7 +157,14 @@ func (v Validator) ValidateLabels(ctx validationContext, ls labels.Labels, strea validation.DiscardedSamples.WithLabelValues(validation.MissingLabels, ctx.userID).Inc() return fmt.Errorf(validation.MissingLabelsErrorMsg) } + numLabelNames := len(ls) + // This is a special case that's often added by the Loki infrastructure. It may result in allowing one extra label + // if incoming requests already have a service_name + if ls.Has(push.LabelServiceName) { + numLabelNames-- + } + if numLabelNames > ctx.maxLabelNamesPerSeries { updateMetrics(validation.MaxLabelNamesPerSeries, ctx.userID, stream) return fmt.Errorf(validation.MaxLabelNamesPerSeriesErrorMsg, stream.Labels, numLabelNames, ctx.maxLabelNamesPerSeries) diff --git a/pkg/ingester-rf1/flush.go b/pkg/ingester-rf1/flush.go index aa22166d4fd3e..2d194a12f5574 100644 --- a/pkg/ingester-rf1/flush.go +++ b/pkg/ingester-rf1/flush.go @@ -96,8 +96,10 @@ func (i *Ingester) flush(l log.Logger, j int, it *wal.PendingSegment) error { } func (i *Ingester) flushSegment(ctx context.Context, j int, w *wal.SegmentWriter) error { - start := time.Now() + ctx, cancelFunc := context.WithTimeout(ctx, i.cfg.FlushOpTimeout) + defer cancelFunc() + start := time.Now() i.metrics.flushesTotal.Add(1) defer func() { i.metrics.flushDuration.Observe(time.Since(start).Seconds()) }() @@ -112,7 +114,7 @@ func (i *Ingester) flushSegment(ctx context.Context, j int, w *wal.SegmentWriter wal.ReportSegmentStats(stats, i.metrics.segmentMetrics) id := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader).String() - if err := i.store.PutObject(ctx, fmt.Sprintf("loki-v2/wal/anon/"+id), buf); err != nil { + if err := i.store.PutObject(ctx, fmt.Sprintf(wal.Dir+id), buf); err != nil { i.metrics.flushFailuresTotal.Inc() return fmt.Errorf("failed to put object: %w", err) } diff --git a/pkg/ingester-rf1/ingester.go b/pkg/ingester-rf1/ingester.go index 0b5a6c5fd724a..8ee0d0e8928b3 100644 --- a/pkg/ingester-rf1/ingester.go +++ b/pkg/ingester-rf1/ingester.go @@ -110,7 +110,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.FlushOpBackoff.MinBackoff, "ingester-rf1.flush-op-backoff-min-period", 100*time.Millisecond, "Minimum backoff period when a flush fails. Each concurrent flush has its own backoff, see `ingester.concurrent-flushes`.") f.DurationVar(&cfg.FlushOpBackoff.MaxBackoff, "ingester-rf1.flush-op-backoff-max-period", time.Minute, "Maximum backoff period when a flush fails. Each concurrent flush has its own backoff, see `ingester.concurrent-flushes`.") f.IntVar(&cfg.FlushOpBackoff.MaxRetries, "ingester-rf1.flush-op-backoff-retries", 10, "Maximum retries for failed flushes.") - f.DurationVar(&cfg.FlushOpTimeout, "ingester-rf1.flush-op-timeout", 10*time.Minute, "The timeout for an individual flush. Will be retried up to `flush-op-backoff-retries` times.") + f.DurationVar(&cfg.FlushOpTimeout, "ingester-rf1.flush-op-timeout", 10*time.Second, "The timeout for an individual flush. Will be retried up to `flush-op-backoff-retries` times.") f.DurationVar(&cfg.MaxSegmentAge, "ingester-rf1.max-segment-age", 500*time.Millisecond, "The maximum age of a segment before it should be flushed. Increasing this value allows more time for a segment to grow to max-segment-size, but may increase latency if the write volume is too small.") f.IntVar(&cfg.MaxSegmentSize, "ingester-rf1.max-segment-size", 8*1024*1024, "The maximum size of a segment before it should be flushed. It is not a strict limit, and segments can exceed the maximum size when individual appends are larger than the remaining capacity.") f.IntVar(&cfg.MaxSegments, "ingester-rf1.max-segments", 10, "The maximum number of segments to buffer in-memory. Increasing this value allows for large bursts of writes to be buffered in memory, but may increase latency if the write volume exceeds the rate at which segments can be flushed.") diff --git a/pkg/ingester-rf1/metastore/client/client.go b/pkg/ingester-rf1/metastore/client/client.go index 6112b7491135f..dacb99f545634 100644 --- a/pkg/ingester-rf1/metastore/client/client.go +++ b/pkg/ingester-rf1/metastore/client/client.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/grafana/dskit/grpcclient" - "github.com/grafana/dskit/instrument" "github.com/grafana/dskit/middleware" "github.com/grafana/dskit/services" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" @@ -14,7 +13,6 @@ import ( "google.golang.org/grpc" "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" - "github.com/grafana/loki/v3/pkg/util/constants" ) type Config struct { @@ -58,10 +56,10 @@ func (c *Client) Service() services.Service { return c.service } func dial(cfg Config, r prometheus.Registerer) (*grpc.ClientConn, error) { latency := prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: constants.Loki, - Name: "metastore_request_duration_seconds", - Help: "Time (in seconds) spent serving requests when using the metastore", - Buckets: instrument.DefBuckets, + Name: "loki_metastore_request_duration_seconds", + Help: "Time (in seconds) spent serving requests when using the metastore", + Buckets: prometheus.ExponentialBuckets(0.001, 4, 8), + NativeHistogramBucketFactor: 1.1, }, []string{"operation", "status_code"}) if r != nil { err := r.Register(latency) diff --git a/pkg/ingester-rf1/metastore/metastore_state_add_block.go b/pkg/ingester-rf1/metastore/metastore_state_add_block.go index ddf8eb2732763..ef2baaf346311 100644 --- a/pkg/ingester-rf1/metastore/metastore_state_add_block.go +++ b/pkg/ingester-rf1/metastore/metastore_state_add_block.go @@ -7,7 +7,7 @@ import ( "github.com/gogo/protobuf/proto" "go.etcd.io/bbolt" - metastorepb "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" + "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" ) func (m *Metastore) AddBlock(_ context.Context, req *metastorepb.AddBlockRequest) (*metastorepb.AddBlockResponse, error) { @@ -29,7 +29,7 @@ func (m *metastoreState) applyAddBlock(request *metastorepb.AddBlockRequest) (*m if err != nil { return nil, err } - err = m.db.boltdb.Update(func(tx *bbolt.Tx) error { + err = m.db.boltdb.Batch(func(tx *bbolt.Tx) error { return updateBlockMetadataBucket(tx, func(bucket *bbolt.Bucket) error { return bucket.Put([]byte(request.Block.Id), value) }) diff --git a/pkg/ingester-rf1/metastore/metastore_state_test.go b/pkg/ingester-rf1/metastore/metastore_state_test.go new file mode 100644 index 0000000000000..ffe4e3217b3ca --- /dev/null +++ b/pkg/ingester-rf1/metastore/metastore_state_test.go @@ -0,0 +1,61 @@ +package metastore + +import ( + "crypto/rand" + "sync" + "testing" + "time" + + "github.com/oklog/ulid" + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" + util_log "github.com/grafana/loki/v3/pkg/util/log" +) + +func Benchmark_metastoreState_applyAddBlock(t *testing.B) { + anHourAgo := ulid.Timestamp(time.Now()) + workers := 1000 + workChan := make(chan struct{}, workers) + wg := sync.WaitGroup{} + m := &metastoreState{ + logger: util_log.Logger, + segmentsMutex: sync.Mutex{}, + segments: make(map[string]*metastorepb.BlockMeta), + db: &boltdb{ + logger: util_log.Logger, + config: Config{ + DataDir: t.TempDir(), + Raft: RaftConfig{ + Dir: t.TempDir(), + }, + }, + }, + } + + err := m.db.open(false) + require.NoError(t, err) + + // Start workers + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + for range workChan { + _, err = m.applyAddBlock(&metastorepb.AddBlockRequest{ + Block: &metastorepb.BlockMeta{ + Id: ulid.MustNew(anHourAgo, rand.Reader).String(), + }, + }) + require.NoError(t, err) + } + wg.Done() + }() + } + + t.ResetTimer() + for i := 0; i < t.N; i++ { + workChan <- struct{}{} + } + close(workChan) + wg.Wait() +} diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index e6e22f72f097e..592ec0690b6b3 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -2,11 +2,13 @@ package ingester import ( "bytes" + "errors" "fmt" "net/http" "sync" "time" + "github.com/dustin/go-humanize" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/backoff" @@ -44,6 +46,60 @@ const ( flushReasonSynced = "synced" ) +// I don't know if this needs to be private but I only needed it in this package. +type flushReasonCounter struct { + flushReasonIdle int + flushReasonMaxAge int + flushReasonForced int + flushReasonNotOwned int + flushReasonFull int + flushReasonSynced int +} + +func (f *flushReasonCounter) Log() []interface{} { + // return counters only if they are non zero + var log []interface{} + if f.flushReasonIdle > 0 { + log = append(log, "idle", f.flushReasonIdle) + } + if f.flushReasonMaxAge > 0 { + log = append(log, "max_age", f.flushReasonMaxAge) + } + if f.flushReasonForced > 0 { + log = append(log, "forced", f.flushReasonForced) + } + if f.flushReasonNotOwned > 0 { + log = append(log, "not_owned", f.flushReasonNotOwned) + } + if f.flushReasonFull > 0 { + log = append(log, "full", f.flushReasonFull) + } + if f.flushReasonSynced > 0 { + log = append(log, "synced", f.flushReasonSynced) + } + return log +} + +func (f *flushReasonCounter) IncrementForReason(reason string) error { + switch reason { + case flushReasonIdle: + f.flushReasonIdle++ + case flushReasonMaxAge: + f.flushReasonMaxAge++ + case flushReasonForced: + f.flushReasonForced++ + case flushReasonNotOwned: + f.flushReasonNotOwned++ + case flushReasonFull: + f.flushReasonFull++ + case flushReasonSynced: + f.flushReasonSynced++ + default: + return fmt.Errorf("unknown reason: %s", reason) + } + return nil +} + // Note: this is called both during the WAL replay (zero or more times) // and then after replay as well. func (i *Ingester) InitFlushQueues() { @@ -62,7 +118,7 @@ func (i *Ingester) Flush() { } // TransferOut implements ring.FlushTransferer -// Noop implemenetation because ingesters have a WAL now that does not require transferring chunks any more. +// Noop implementation because ingesters have a WAL now that does not require transferring chunks any more. // We return ErrTransferDisabled to indicate that we don't support transfers, and therefore we may flush on shutdown if configured to do so. func (i *Ingester) TransferOut(_ context.Context) error { return ring.ErrTransferDisabled @@ -179,7 +235,6 @@ func (i *Ingester) flushLoop(j int) { m := util_log.WithUserID(op.userID, l) err := i.flushOp(m, op) - if err != nil { level.Error(m).Log("msg", "failed to flush", "err", err) } @@ -220,8 +275,33 @@ func (i *Ingester) flushUserSeries(ctx context.Context, userID string, fp model. return nil } + totalCompressedSize := 0 + totalUncompressedSize := 0 + frc := flushReasonCounter{} + for _, c := range chunks { + totalCompressedSize += c.chunk.CompressedSize() + totalUncompressedSize += c.chunk.UncompressedSize() + err := frc.IncrementForReason(c.reason) + if err != nil { + level.Error(i.logger).Log("msg", "error incrementing flush reason", "err", err) + } + } + lbs := labels.String() - level.Info(i.logger).Log("msg", "flushing stream", "user", userID, "fp", fp, "immediate", immediate, "num_chunks", len(chunks), "labels", lbs) + logValues := make([]interface{}, 0, 35) + logValues = append(logValues, + "msg", "flushing stream", + "user", userID, + "fp", fp, + "immediate", immediate, + "num_chunks", len(chunks), + "total_comp", humanize.Bytes(uint64(totalCompressedSize)), + "avg_comp", humanize.Bytes(uint64(totalCompressedSize/len(chunks))), + "total_uncomp", humanize.Bytes(uint64(totalUncompressedSize)), + "avg_uncomp", humanize.Bytes(uint64(totalUncompressedSize/len(chunks)))) + logValues = append(logValues, frc.Log()...) + logValues = append(logValues, "labels", lbs) + level.Info(i.logger).Log(logValues...) ctx = user.InjectOrgID(ctx, userID) ctx, cancelFunc := context.WithTimeout(ctx, i.cfg.FlushOpTimeout) @@ -410,10 +490,15 @@ func (i *Ingester) encodeChunk(ctx context.Context, ch *chunk.Chunk, desc *chunk } start := time.Now() chunkBytesSize := desc.chunk.BytesSize() + 4*1024 // size + 4kB should be enough room for cortex header - if err := ch.EncodeTo(bytes.NewBuffer(make([]byte, 0, chunkBytesSize))); err != nil { - return fmt.Errorf("chunk encoding: %w", err) + if err := ch.EncodeTo(bytes.NewBuffer(make([]byte, 0, chunkBytesSize)), i.logger); err != nil { + if !errors.Is(err, chunk.ErrChunkDecode) { + return fmt.Errorf("chunk encoding: %w", err) + } + + i.metrics.chunkDecodeFailures.WithLabelValues(ch.UserID).Inc() } i.metrics.chunkEncodeTime.Observe(time.Since(start).Seconds()) + i.metrics.chunksEncoded.WithLabelValues(ch.UserID).Inc() return nil } diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index ad190285ccd08..ff4db43747676 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -50,6 +50,8 @@ type ingesterMetrics struct { chunksFlushFailures prometheus.Counter chunksFlushedPerReason *prometheus.CounterVec chunkLifespan prometheus.Histogram + chunksEncoded *prometheus.CounterVec + chunkDecodeFailures *prometheus.CounterVec flushedChunksStats *analytics.Counter flushedChunksBytesStats *analytics.Statistics flushedChunksLinesStats *analytics.Statistics @@ -252,12 +254,28 @@ func newIngesterMetrics(r prometheus.Registerer, metricsNamespace string) *inges // 1h -> 8hr Buckets: prometheus.LinearBuckets(1, 1, 8), }), - flushedChunksStats: analytics.NewCounter("ingester_flushed_chunks"), - flushedChunksBytesStats: analytics.NewStatistics("ingester_flushed_chunks_bytes"), - flushedChunksLinesStats: analytics.NewStatistics("ingester_flushed_chunks_lines"), - flushedChunksAgeStats: analytics.NewStatistics("ingester_flushed_chunks_age_seconds"), - flushedChunksLifespanStats: analytics.NewStatistics("ingester_flushed_chunks_lifespan_seconds"), - flushedChunksUtilizationStats: analytics.NewStatistics("ingester_flushed_chunks_utilization"), + chunksEncoded: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "ingester_chunks_encoded_total", + Help: "The total number of chunks encoded in the ingester.", + }, []string{"user"}), + chunkDecodeFailures: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "ingester_chunk_decode_failures_total", + Help: "The number of freshly encoded chunks that failed to decode.", + }, []string{"user"}), + flushedChunksStats: analytics.NewCounter("ingester_flushed_chunks"), + flushedChunksBytesStats: analytics.NewStatistics("ingester_flushed_chunks_bytes"), + flushedChunksLinesStats: analytics.NewStatistics("ingester_flushed_chunks_lines"), + flushedChunksAgeStats: analytics.NewStatistics( + "ingester_flushed_chunks_age_seconds", + ), + flushedChunksLifespanStats: analytics.NewStatistics( + "ingester_flushed_chunks_lifespan_seconds", + ), + flushedChunksUtilizationStats: analytics.NewStatistics( + "ingester_flushed_chunks_utilization", + ), chunksCreatedTotal: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "ingester_chunks_created_total", diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index c63b32c6111bb..a9b174952f286 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -10,6 +10,8 @@ import ( "net/http" "time" + "github.com/grafana/loki/v3/pkg/logql/syntax" + "github.com/go-kit/log/level" "github.com/grafana/loki/pkg/push" @@ -25,7 +27,6 @@ import ( "github.com/grafana/loki/v3/pkg/analytics" "github.com/grafana/loki/v3/pkg/loghttp" "github.com/grafana/loki/v3/pkg/logproto" - "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/util" "github.com/grafana/loki/v3/pkg/util/constants" "github.com/grafana/loki/v3/pkg/util/unmarshal" @@ -57,7 +58,11 @@ var ( linesReceivedStats = analytics.NewCounter("distributor_lines_received") ) -const applicationJSON = "application/json" +const ( + applicationJSON = "application/json" + LabelServiceName = "service_name" + ServiceUnknown = "unknown_service" +) type TenantsRetention interface { RetentionPeriodFor(userID string, lbs labels.Labels) time.Duration @@ -65,6 +70,7 @@ type TenantsRetention interface { type Limits interface { OTLPConfig(userID string) OTLPConfig + DiscoverServiceName(userID string) []string } type EmptyLimits struct{} @@ -73,6 +79,10 @@ func (EmptyLimits) OTLPConfig(string) OTLPConfig { return DefaultOTLPConfig(GlobalOTLPConfig{}) } +func (EmptyLimits) DiscoverServiceName(string) []string { + return nil +} + type RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) type RequestParserWrapper func(inner RequestParser) RequestParser @@ -148,7 +158,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete return req, nil } -func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, _ Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { +func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { // Body var body io.Reader // bodySize should always reflect the compressed size of the request body @@ -217,16 +227,33 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.ContentType = contentType pushStats.ContentEncoding = contentEncoding - for _, s := range req.Streams { + discoverServiceName := limits.DiscoverServiceName(userID) + for i := range req.Streams { + s := req.Streams[i] pushStats.StreamLabelsSize += int64(len(s.Labels)) - var lbs labels.Labels - if tenantsRetention != nil || tracker != nil { - lbs, err = syntax.ParseLabels(s.Labels) - if err != nil { - return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) + lbs, err := syntax.ParseLabels(s.Labels) + if err != nil { + return nil, nil, fmt.Errorf("couldn't parse labels: %w", err) + } + + if !lbs.Has(LabelServiceName) && len(discoverServiceName) > 0 { + serviceName := ServiceUnknown + for _, labelName := range discoverServiceName { + if labelVal := lbs.Get(labelName); labelVal != "" { + serviceName = labelVal + break + } } + + lb := labels.NewBuilder(lbs) + lbs = lb.Set(LabelServiceName, serviceName).Labels() + s.Labels = lbs.String() + + // Remove the added label after it's added to the stream so it's not consumed by subsequent steps + lbs = lb.Del(LabelServiceName).Labels() } + var retentionPeriod time.Duration if tenantsRetention != nil { retentionPeriod = tenantsRetention.RetentionPeriodFor(userID, lbs) @@ -249,6 +276,8 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.MostRecentEntryTimestamp = e.Timestamp } } + + req.Streams[i] = s } return &req, pushStats, nil diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index ac83492d62eba..0484afe31c3b0 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -54,10 +54,12 @@ func TestParseRequest(t *testing.T) { contentType string contentEncoding string valid bool + enableServiceDiscovery bool expectedStructuredMetadataBytes int expectedBytes int expectedLines int expectedBytesUsageTracker map[string]float64 + expectedLabels labels.Labels }{ { path: `/loki/api/v1/push`, @@ -79,6 +81,7 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), }, { path: `/loki/api/v1/push`, @@ -89,6 +92,7 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), }, { path: `/loki/api/v1/push`, @@ -106,6 +110,7 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), }, { path: `/loki/api/v1/push`, @@ -116,6 +121,7 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), }, { path: `/loki/api/v1/push`, @@ -133,6 +139,7 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), }, { path: `/loki/api/v1/push`, @@ -143,6 +150,7 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), }, { path: `/loki/api/v1/push`, @@ -196,6 +204,29 @@ func TestParseRequest(t *testing.T) { expectedBytes: len("fizzbuzz") + 2*len("a") + 2*len("b"), expectedLines: 1, expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuzz") + 2*len("a") + 2*len("b"))}, + expectedLabels: labels.FromStrings("foo", "bar2"), + }, + { + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2", "job": "stuff" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + valid: true, + enableServiceDiscovery: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2", job="stuff"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2", "job", "stuff", LabelServiceName, "stuff"), + }, + { + path: `/loki/api/v1/push`, + body: `{"streams": [{ "stream": { "foo": "bar2" }, "values": [ [ "1570818238000000000", "fizzbuzz" ] ] }]}`, + contentType: `application/json`, + valid: true, + enableServiceDiscovery: true, + expectedBytes: len("fizzbuzz"), + expectedLines: 1, + expectedBytesUsageTracker: map[string]float64{`{foo="bar2"}`: float64(len("fizzbuss"))}, + expectedLabels: labels.FromStrings("foo", "bar2", LabelServiceName, ServiceUnknown), }, } { t.Run(fmt.Sprintf("test %d", index), func(t *testing.T) { @@ -212,7 +243,7 @@ func TestParseRequest(t *testing.T) { } tracker := NewMockTracker() - data, err := ParseRequest(util_log.Logger, "fake", request, nil, nil, ParseLokiRequest, tracker) + data, err := ParseRequest(util_log.Logger, "fake", request, nil, &fakeLimits{test.enableServiceDiscovery}, ParseLokiRequest, tracker) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -231,6 +262,7 @@ func TestParseRequest(t *testing.T) { require.Equal(t, float64(test.expectedStructuredMetadataBytes), testutil.ToFloat64(structuredMetadataBytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedBytes), testutil.ToFloat64(bytesIngested.WithLabelValues("fake", ""))) require.Equal(t, float64(test.expectedLines), testutil.ToFloat64(linesIngested.WithLabelValues("fake"))) + require.Equal(t, test.expectedLabels.String(), data.Streams[0].Labels) require.InDeltaMapValuesf(t, test.expectedBytesUsageTracker, tracker.receivedBytes, 0.0, "%s != %s", test.expectedBytesUsageTracker, tracker.receivedBytes) } else { assert.Errorf(t, err, "Should give error for %d", index) @@ -246,6 +278,33 @@ func TestParseRequest(t *testing.T) { } } +type fakeLimits struct { + enabled bool +} + +func (l *fakeLimits) OTLPConfig(_ string) OTLPConfig { + return OTLPConfig{} +} + +func (l *fakeLimits) DiscoverServiceName(_ string) []string { + if !l.enabled { + return nil + } + + return []string{ + "service", + "app", + "application", + "name", + "app_kubernetes_io_name", + "container", + "container_name", + "component", + "workload", + "job", + } +} + type MockCustomTracker struct { receivedBytes map[string]float64 discardedBytes map[string]float64 diff --git a/pkg/logql/syntax/ast.go b/pkg/logql/syntax/ast.go index 2e7b8a2e0e19d..e391291c23242 100644 --- a/pkg/logql/syntax/ast.go +++ b/pkg/logql/syntax/ast.go @@ -185,16 +185,22 @@ func (m MultiStageExpr) reorderStages() []StageExpr { func combineFilters(in []*LineFilterExpr) StageExpr { result := in[len(in)-1] for i := len(in) - 2; i >= 0; i-- { - leafNode(result).Left = in[i] + leaf := leafNode(result, in[i]) + if leaf != nil { + leaf.Left = in[i] + } } return result } -func leafNode(in *LineFilterExpr) *LineFilterExpr { +func leafNode(in *LineFilterExpr, child *LineFilterExpr) *LineFilterExpr { current := in //nolint:revive for ; current.Left != nil; current = current.Left { + if current == child || current.Left == child { + return nil + } } return current } diff --git a/pkg/logql/syntax/ast_test.go b/pkg/logql/syntax/ast_test.go index 62feb88636c2c..b9f6dcdc46bd8 100644 --- a/pkg/logql/syntax/ast_test.go +++ b/pkg/logql/syntax/ast_test.go @@ -1094,3 +1094,24 @@ func TestGroupingString(t *testing.T) { } require.Equal(t, " without ()", g.String()) } + +func TestCombineFilters(t *testing.T) { + in := []*LineFilterExpr{ + {LineFilter: LineFilter{Ty: log.LineMatchEqual, Match: "test1"}}, + {LineFilter: LineFilter{Ty: log.LineMatchEqual, Match: "test2"}}, + } + + var combineFilter StageExpr + for i := 0; i < 2; i++ { + combineFilter = combineFilters(in) + } + + current := combineFilter.(*LineFilterExpr) + i := 0 + for ; current.Left != nil; current = current.Left { + i++ + if i > 2 { + t.Fatalf("left num isn't a correct number") + } + } +} diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 1ede7ee806b79..d823f5cedb5cd 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -52,6 +52,7 @@ import ( metastoreclient "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/client" "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/health" "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" + "github.com/grafana/loki/v3/pkg/ingester-rf1/objstore" "github.com/grafana/loki/v3/pkg/logproto" "github.com/grafana/loki/v3/pkg/logql" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" @@ -415,7 +416,11 @@ func (t *Loki) initQuerier() (services.Service, error) { if t.Cfg.QuerierRF1.Enabled { logger.Log("Using RF-1 querier implementation") - t.Querier, err = querierrf1.New(t.Cfg.QuerierRF1, t.Store, t.Overrides, deleteStore, logger) + store, err := objstore.New(t.Cfg.SchemaConfig.Configs, t.Cfg.StorageConfig, t.ClientMetrics) + if err != nil { + return nil, err + } + t.Querier, err = querierrf1.New(t.Cfg.QuerierRF1, t.Store, t.Overrides, deleteStore, t.MetastoreClient, store, logger) if err != nil { return nil, err } @@ -1818,7 +1823,7 @@ func (t *Loki) initMetastore() (services.Service, error) { return nil, nil } if t.Cfg.isTarget(All) { - t.Cfg.MetastoreClient.MetastoreAddress = fmt.Sprintf("localhost:%s", t.Cfg.Server.GRPCListenAddress) + t.Cfg.MetastoreClient.MetastoreAddress = fmt.Sprintf("localhost:%d", t.Cfg.Server.GRPCListenPort) } m, err := metastore.New(t.Cfg.Metastore, log.With(util_log.Logger, "component", "metastore"), prometheus.DefaultRegisterer, t.health) if err != nil { diff --git a/pkg/querier-rf1/querier.go b/pkg/querier-rf1/querier.go index 9504fe23482ab..c4a9dd76ba5f6 100644 --- a/pkg/querier-rf1/querier.go +++ b/pkg/querier-rf1/querier.go @@ -34,6 +34,7 @@ import ( "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/logqlmodel" "github.com/grafana/loki/v3/pkg/querier" + "github.com/grafana/loki/v3/pkg/querier-rf1/wal" querier_limits "github.com/grafana/loki/v3/pkg/querier/limits" "github.com/grafana/loki/v3/pkg/querier/plan" "github.com/grafana/loki/v3/pkg/storage" @@ -97,6 +98,7 @@ type Rf1Querier struct { deleteGetter deleteGetter logger log.Logger patternQuerier PatterQuerier + walQuerier logql.Querier } type deleteGetter interface { @@ -104,12 +106,17 @@ type deleteGetter interface { } // New makes a new Querier for RF1 work. -func New(cfg Config, store Store, limits Limits, d deleteGetter, logger log.Logger) (*Rf1Querier, error) { +func New(cfg Config, store Store, limits Limits, d deleteGetter, metastore wal.Metastore, b wal.BlockStorage, logger log.Logger) (*Rf1Querier, error) { + querier, err := wal.New(metastore, b) + if err != nil { + return nil, err + } return &Rf1Querier{ cfg: cfg, store: store, limits: limits, deleteGetter: d, + walQuerier: querier, logger: logger, }, nil } @@ -134,7 +141,7 @@ func (q *Rf1Querier) SelectLogs(ctx context.Context, params logql.SelectLogParam "msg", "querying rf1 store", "params", params) } - storeIter, err := q.store.SelectLogs(ctx, params) + storeIter, err := q.walQuerier.SelectLogs(ctx, params) if err != nil { return nil, err } @@ -164,7 +171,7 @@ func (q *Rf1Querier) SelectSamples(ctx context.Context, params logql.SelectSampl "msg", "querying rf1 store for samples", "params", params) } - storeIter, err := q.store.SelectSamples(ctx, params) + storeIter, err := q.walQuerier.SelectSamples(ctx, params) if err != nil { return nil, err } diff --git a/pkg/querier-rf1/wal/chunks.go b/pkg/querier-rf1/wal/chunks.go new file mode 100644 index 0000000000000..76070006aff40 --- /dev/null +++ b/pkg/querier-rf1/wal/chunks.go @@ -0,0 +1,323 @@ +package wal + +import ( + "context" + "fmt" + "sort" + + "github.com/prometheus/prometheus/model/labels" + "golang.org/x/sync/errgroup" + + "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql/log" + "github.com/grafana/loki/v3/pkg/storage/wal" + "github.com/grafana/loki/v3/pkg/storage/wal/chunks" + "github.com/grafana/loki/v3/pkg/storage/wal/index" + + "github.com/grafana/loki/pkg/push" +) + +const defaultBatchSize = 16 + +type ChunkData struct { + meta *chunks.Meta + labels labels.Labels + id string +} + +func newChunkData(id string, lbs *labels.ScratchBuilder, meta *chunks.Meta) ChunkData { + lbs.Sort() + newLbs := lbs.Labels() + j := 0 + for _, l := range newLbs { + if l.Name != index.TenantLabel { + newLbs[j] = l + j++ + } + } + newLbs = newLbs[:j] + return ChunkData{ + id: id, + meta: &chunks.Meta{ // incoming Meta is from a shared buffer, so create a new one + Ref: meta.Ref, + MinTime: meta.MinTime, + MaxTime: meta.MaxTime, + }, + labels: newLbs, + } +} + +// ChunksEntryIterator iterates over log entries +type ChunksEntryIterator[T iter.EntryIterator] struct { + baseChunksIterator[T] +} + +// ChunksSampleIterator iterates over metric samples +type ChunksSampleIterator[T iter.SampleIterator] struct { + baseChunksIterator[T] +} + +func NewChunksEntryIterator( + ctx context.Context, + storage BlockStorage, + chunks []ChunkData, + pipeline log.Pipeline, + direction logproto.Direction, + minT, maxT int64, +) *ChunksEntryIterator[iter.EntryIterator] { + sortChunks(chunks, direction) + return &ChunksEntryIterator[iter.EntryIterator]{ + baseChunksIterator: baseChunksIterator[iter.EntryIterator]{ + ctx: ctx, + chunks: chunks, + direction: direction, + storage: storage, + bachSize: defaultBatchSize, + batch: make([]ChunkData, 0, defaultBatchSize), + minT: minT, + maxT: maxT, + + iteratorFactory: func(chunks []ChunkData) (iter.EntryIterator, error) { + return createNextEntryIterator(ctx, chunks, direction, pipeline, storage, minT, maxT) + }, + isNil: func(it iter.EntryIterator) bool { return it == nil }, + }, + } +} + +func NewChunksSampleIterator( + ctx context.Context, + storage BlockStorage, + chunks []ChunkData, + extractor log.SampleExtractor, + minT, maxT int64, +) *ChunksSampleIterator[iter.SampleIterator] { + sortChunks(chunks, logproto.FORWARD) + return &ChunksSampleIterator[iter.SampleIterator]{ + baseChunksIterator: baseChunksIterator[iter.SampleIterator]{ + ctx: ctx, + chunks: chunks, + direction: logproto.FORWARD, + storage: storage, + bachSize: defaultBatchSize, + batch: make([]ChunkData, 0, defaultBatchSize), + minT: minT, + maxT: maxT, + + iteratorFactory: func(chunks []ChunkData) (iter.SampleIterator, error) { + return createNextSampleIterator(ctx, chunks, extractor, storage, minT, maxT) + }, + isNil: func(it iter.SampleIterator) bool { return it == nil }, + }, + } +} + +func sortChunks(chunks []ChunkData, direction logproto.Direction) { + sort.Slice(chunks, func(i, j int) bool { + if direction == logproto.FORWARD { + t1, t2 := chunks[i].meta.MinTime, chunks[j].meta.MinTime + if t1 != t2 { + return t1 < t2 + } + return labels.Compare(chunks[i].labels, chunks[j].labels) < 0 + } + t1, t2 := chunks[i].meta.MaxTime, chunks[j].meta.MaxTime + if t1 != t2 { + return t1 > t2 + } + return labels.Compare(chunks[i].labels, chunks[j].labels) < 0 + }) +} + +// baseChunksIterator contains common fields and methods for both entry and sample iterators +type baseChunksIterator[T interface { + Next() bool + Close() error + Err() error + StreamHash() uint64 + Labels() string +}] struct { + chunks []ChunkData + direction logproto.Direction + minT, maxT int64 + storage BlockStorage + ctx context.Context + iteratorFactory func([]ChunkData) (T, error) + isNil func(T) bool + + bachSize int + batch []ChunkData + current T + err error +} + +func (b *baseChunksIterator[T]) nextBatch() error { + b.batch = b.batch[:0] + for len(b.chunks) > 0 && + (len(b.batch) < b.bachSize || + isOverlapping(b.batch[len(b.batch)-1], b.chunks[0], b.direction)) { + b.batch = append(b.batch, b.chunks[0]) + b.chunks = b.chunks[1:] + } + // todo: error if the batch is too big. + return nil +} + +// todo: better chunk batch iterator +func (b *baseChunksIterator[T]) Next() bool { + for b.isNil(b.current) || !b.current.Next() { + if !b.isNil(b.current) { + if err := b.current.Close(); err != nil { + b.err = err + return false + } + } + if len(b.chunks) == 0 { + return false + } + if err := b.nextBatch(); err != nil { + b.err = err + return false + } + var err error + b.current, err = b.iteratorFactory(b.batch) + if err != nil { + b.err = err + return false + } + } + return true +} + +func createNextEntryIterator( + ctx context.Context, + batch []ChunkData, + direction logproto.Direction, + pipeline log.Pipeline, + storage BlockStorage, + minT, maxT int64, +) (iter.EntryIterator, error) { + iterators := make([]iter.EntryIterator, 0, len(batch)) + + data, err := downloadChunks(ctx, storage, batch) + if err != nil { + return nil, err + } + + for i, chunk := range batch { + streamPipeline := pipeline.ForStream(chunk.labels) + chunkIterator, err := chunks.NewEntryIterator(data[i], streamPipeline, direction, minT, maxT) + if err != nil { + return nil, fmt.Errorf("error creating entry iterator: %w", err) + } + iterators = append(iterators, chunkIterator) + } + + // todo: Use NonOverlapping iterator when possible. This will reduce the amount of entries processed during iteration. + return iter.NewSortEntryIterator(iterators, direction), nil +} + +func createNextSampleIterator( + ctx context.Context, + batch []ChunkData, + pipeline log.SampleExtractor, + storage BlockStorage, + minT, maxT int64, +) (iter.SampleIterator, error) { + iterators := make([]iter.SampleIterator, 0, len(batch)) + + data, err := downloadChunks(ctx, storage, batch) + if err != nil { + return nil, err + } + + for i, chunk := range batch { + streamPipeline := pipeline.ForStream(chunk.labels) + chunkIterator, err := chunks.NewSampleIterator(data[i], streamPipeline, minT, maxT) + if err != nil { + return nil, fmt.Errorf("error creating sample iterator: %w", err) + } + iterators = append(iterators, chunkIterator) + } + + return iter.NewSortSampleIterator(iterators), nil +} + +func (b *baseChunksIterator[T]) Close() error { + if !b.isNil(b.current) { + return b.current.Close() + } + return nil +} + +func (b *baseChunksIterator[T]) Err() error { + if b.err != nil { + return b.err + } + if !b.isNil(b.current) { + return b.current.Err() + } + return nil +} + +func (b *baseChunksIterator[T]) Labels() string { + return b.current.Labels() +} + +func (b *baseChunksIterator[T]) StreamHash() uint64 { + return b.current.StreamHash() +} + +func (c *ChunksEntryIterator[T]) At() push.Entry { return c.current.At() } +func (c *ChunksSampleIterator[T]) At() logproto.Sample { return c.current.At() } + +func isOverlapping(first, second ChunkData, direction logproto.Direction) bool { + if direction == logproto.BACKWARD { + return first.meta.MinTime <= second.meta.MaxTime + } + return first.meta.MaxTime >= second.meta.MinTime +} + +func downloadChunks(ctx context.Context, storage BlockStorage, chks []ChunkData) ([][]byte, error) { + data := make([][]byte, len(chks)) + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(64) + for i, chunk := range chks { + chunk := chunk + i := i + g.Go(func() error { + chunkData, err := readChunkData(ctx, storage, chunk) + if err != nil { + return fmt.Errorf("error reading chunk data: %w", err) + } + data[i] = chunkData + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + return data, nil +} + +func readChunkData(ctx context.Context, storage BlockStorage, chunk ChunkData) ([]byte, error) { + offset, size := chunk.meta.Ref.Unpack() + // todo: We should be able to avoid many IOPS to object storage + // if chunks are next to each other and we should be able to pack range request + // together. + reader, err := storage.GetObjectRange(ctx, wal.Dir+chunk.id, int64(offset), int64(size)) + if err != nil { + return nil, err + } + defer reader.Close() + + data := make([]byte, size) + _, err = reader.Read(data) + if err != nil { + return nil, err + } + + return data, nil +} diff --git a/pkg/querier-rf1/wal/chunks_test.go b/pkg/querier-rf1/wal/chunks_test.go new file mode 100644 index 0000000000000..0d0192c04b17a --- /dev/null +++ b/pkg/querier-rf1/wal/chunks_test.go @@ -0,0 +1,516 @@ +package wal + +import ( + "bytes" + "context" + "fmt" + "io" + "testing" + "time" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql/syntax" + "github.com/grafana/loki/v3/pkg/storage/wal" + walchunks "github.com/grafana/loki/v3/pkg/storage/wal/chunks" +) + +type mockBlockStorage struct { + data map[string][]byte +} + +func (m *mockBlockStorage) GetObjectRange(_ context.Context, objectKey string, off, length int64) (io.ReadCloser, error) { + data := m.data[objectKey] + return io.NopCloser(bytes.NewReader(data[off : off+length])), nil +} + +func TestChunksEntryIterator(t *testing.T) { + ctx := context.Background() + storage := &mockBlockStorage{data: make(map[string][]byte)} + + // Generate test data with multiple batches + chunkData := generateTestChunkData(5 * defaultBatchSize) + chks := writeChunksToStorage(t, storage, chunkData) + + tests := []struct { + name string + direction logproto.Direction + start time.Time + end time.Time + expected []logproto.Entry + }{ + { + name: "forward direction, all entries", + direction: logproto.FORWARD, + start: time.Unix(0, 0), + end: time.Unix(int64(5*defaultBatchSize+1), 0), + expected: flattenEntries(chunkData), + }, + { + name: "backward direction, all entries", + direction: logproto.BACKWARD, + start: time.Unix(0, 0), + end: time.Unix(int64(5*defaultBatchSize+1), 0), + expected: reverseEntries(flattenEntries(chunkData)), + }, + { + name: "forward direction, partial range", + direction: logproto.FORWARD, + start: time.Unix(int64(defaultBatchSize), 0), + end: time.Unix(int64(3*defaultBatchSize), 0), + expected: selectEntries(flattenEntries(chunkData), defaultBatchSize, 3*defaultBatchSize), + }, + { + name: "backward direction, partial range", + direction: logproto.BACKWARD, + start: time.Unix(int64(defaultBatchSize), 0), + end: time.Unix(int64(3*defaultBatchSize), 0), + expected: reverseEntries(selectEntries(flattenEntries(chunkData), defaultBatchSize, 3*defaultBatchSize)), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr, err := syntax.ParseLogSelector(`{app=~".+"}`, false) + require.NoError(t, err) + + pipeline, err := expr.Pipeline() + require.NoError(t, err) + + iterator := NewChunksEntryIterator(ctx, storage, chks, pipeline, tt.direction, tt.start.UnixNano(), tt.end.UnixNano()) + + result := iterateEntries(iterator) + require.NoError(t, iterator.Close()) + require.NoError(t, iterator.Err()) + + assertEqualEntries(t, tt.expected, result) + }) + } +} + +func TestChunksSampleIterator(t *testing.T) { + ctx := context.Background() + storage := &mockBlockStorage{data: make(map[string][]byte)} + + // Generate test data with multiple batches + chunkData := generateTestChunkData(5 * defaultBatchSize) + chks := writeChunksToStorage(t, storage, chunkData) + + tests := []struct { + name string + start time.Time + end time.Time + expected []logproto.Sample + }{ + { + name: "all samples", + start: time.Unix(0, 0), + end: time.Unix(int64(5*defaultBatchSize+1), 0), + expected: entriesToSamples(flattenEntries(chunkData)), + }, + { + name: "partial range", + start: time.Unix(int64(defaultBatchSize), 0), + end: time.Unix(int64(3*defaultBatchSize), 0), + expected: entriesToSamples(selectEntries(flattenEntries(chunkData), defaultBatchSize, 3*defaultBatchSize)), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr, err := syntax.ParseSampleExpr(`count_over_time({app=~".+"} [1m])`) + require.NoError(t, err) + + extractor, err := expr.Extractor() + require.NoError(t, err) + iterator := NewChunksSampleIterator(ctx, storage, chks, extractor, tt.start.UnixNano(), tt.end.UnixNano()) + + result := iterateSamples(iterator) + require.NoError(t, iterator.Close()) + require.NoError(t, iterator.Err()) + + assertEqualSamples(t, tt.expected, result) + }) + } +} + +func TestSortChunks(t *testing.T) { + chks := []ChunkData{ + { + meta: &walchunks.Meta{MinTime: 2, MaxTime: 4}, + labels: labels.FromStrings("app", "test1"), + }, + { + meta: &walchunks.Meta{MinTime: 1, MaxTime: 3}, + labels: labels.FromStrings("app", "test2"), + }, + { + meta: &walchunks.Meta{MinTime: 1, MaxTime: 3}, + labels: labels.FromStrings("app", "test1"), + }, + } + + t.Run("forward direction", func(t *testing.T) { + sortChunks(chks, logproto.FORWARD) + require.Equal(t, int64(1), chks[0].meta.MinTime) + require.Equal(t, "test1", chks[0].labels.Get("app")) + require.Equal(t, int64(1), chks[1].meta.MinTime) + require.Equal(t, "test2", chks[1].labels.Get("app")) + require.Equal(t, int64(2), chks[2].meta.MinTime) + }) + + t.Run("backward direction", func(t *testing.T) { + sortChunks(chks, logproto.BACKWARD) + require.Equal(t, int64(4), chks[0].meta.MaxTime) + require.Equal(t, "test1", chks[0].labels.Get("app")) + require.Equal(t, int64(3), chks[1].meta.MaxTime) + require.Equal(t, "test1", chks[1].labels.Get("app")) + require.Equal(t, int64(3), chks[2].meta.MaxTime) + require.Equal(t, "test2", chks[2].labels.Get("app")) + }) +} + +func TestIsOverlapping(t *testing.T) { + tests := []struct { + name string + first ChunkData + second ChunkData + direction logproto.Direction + expected bool + }{ + { + name: "overlapping forward", + first: ChunkData{meta: &walchunks.Meta{MinTime: 1, MaxTime: 3}}, + second: ChunkData{meta: &walchunks.Meta{MinTime: 2, MaxTime: 4}}, + direction: logproto.FORWARD, + expected: true, + }, + { + name: "non-overlapping forward", + first: ChunkData{meta: &walchunks.Meta{MinTime: 1, MaxTime: 2}}, + second: ChunkData{meta: &walchunks.Meta{MinTime: 3, MaxTime: 4}}, + direction: logproto.FORWARD, + expected: false, + }, + { + name: "overlapping backward", + first: ChunkData{meta: &walchunks.Meta{MinTime: 2, MaxTime: 4}}, + second: ChunkData{meta: &walchunks.Meta{MinTime: 1, MaxTime: 3}}, + direction: logproto.BACKWARD, + expected: true, + }, + { + name: "non-overlapping backward", + first: ChunkData{meta: &walchunks.Meta{MinTime: 3, MaxTime: 4}}, + second: ChunkData{meta: &walchunks.Meta{MinTime: 1, MaxTime: 2}}, + direction: logproto.BACKWARD, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isOverlapping(tt.first, tt.second, tt.direction) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestBaseChunkIterator(t *testing.T) { + ctx := context.Background() + + testCases := []struct { + name string + chunks []ChunkData + direction logproto.Direction + expected [][]ChunkData + }{ + { + name: "Forward, non-overlapping", + chunks: []ChunkData{ + newTestChunkData("1", 100, 200), + newTestChunkData("2", 300, 400), + newTestChunkData("3", 500, 600), + newTestChunkData("4", 700, 800), + }, + direction: logproto.FORWARD, + expected: [][]ChunkData{ + {newTestChunkData("1", 100, 200), newTestChunkData("2", 300, 400)}, + {newTestChunkData("3", 500, 600), newTestChunkData("4", 700, 800)}, + }, + }, + { + name: "Backward, non-overlapping", + chunks: []ChunkData{ + newTestChunkData("4", 700, 800), + newTestChunkData("3", 500, 600), + newTestChunkData("2", 300, 400), + newTestChunkData("1", 100, 200), + }, + direction: logproto.BACKWARD, + expected: [][]ChunkData{ + {newTestChunkData("4", 700, 800), newTestChunkData("3", 500, 600)}, + {newTestChunkData("2", 300, 400), newTestChunkData("1", 100, 200)}, + }, + }, + { + name: "Forward, overlapping", + chunks: []ChunkData{ + newTestChunkData("1", 100, 300), + newTestChunkData("2", 200, 400), + newTestChunkData("3", 350, 550), + newTestChunkData("4", 600, 800), + }, + direction: logproto.FORWARD, + expected: [][]ChunkData{ + {newTestChunkData("1", 100, 300), newTestChunkData("2", 200, 400), newTestChunkData("3", 350, 550)}, + {newTestChunkData("4", 600, 800)}, + }, + }, + { + name: "Backward, overlapping", + chunks: []ChunkData{ + newTestChunkData("4", 600, 800), + newTestChunkData("3", 350, 550), + newTestChunkData("2", 200, 400), + newTestChunkData("1", 100, 300), + newTestChunkData("0", 10, 20), + }, + direction: logproto.BACKWARD, + expected: [][]ChunkData{ + {newTestChunkData("4", 600, 800), newTestChunkData("3", 350, 550), newTestChunkData("2", 200, 400), newTestChunkData("1", 100, 300)}, + {newTestChunkData("0", 10, 20)}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + iter := &testBaseChunkIterator{ + baseChunksIterator: baseChunksIterator[*testIterator]{ + ctx: ctx, + chunks: tc.chunks, + direction: tc.direction, + bachSize: 2, + batch: make([]ChunkData, 0, 2), + iteratorFactory: func(chunks []ChunkData) (*testIterator, error) { + return &testIterator{chunks: chunks}, nil + }, + isNil: func(it *testIterator) bool { return it == nil }, + }, + } + var batches [][]ChunkData + for len(iter.chunks) > 0 { + err := iter.nextBatch() + require.NoError(t, err) + + batch := make([]ChunkData, len(iter.batch)) + copy(batch, iter.batch) + batches = append(batches, batch) + } + + require.Equal(t, tc.expected, batches) + }) + } +} + +// Helper functions and types + +type testBaseChunkIterator struct { + baseChunksIterator[*testIterator] +} + +type testIterator struct { + chunks []ChunkData + index int +} + +func (t *testIterator) Next() bool { + t.index++ + return t.index < len(t.chunks) +} + +func (t *testIterator) Close() error { return nil } +func (t *testIterator) Err() error { return nil } +func (t *testIterator) StreamHash() uint64 { return 0 } +func (t *testIterator) Labels() string { return "" } +func (t *testIterator) At() logproto.Entry { return logproto.Entry{} } + +func newTestChunkData(id string, minTime, maxTime int64) ChunkData { + return ChunkData{ + id: id, + meta: &walchunks.Meta{ + MinTime: minTime, + MaxTime: maxTime, + }, + labels: labels.Labels{}, + } +} + +func createChunk(minTime, maxTime int64, labelName, labelValue string) ChunkData { + return ChunkData{ + meta: &walchunks.Meta{ + MinTime: minTime, + MaxTime: maxTime, + }, + labels: labels.FromStrings(labelName, labelValue), + } +} + +func assertEqualChunks(t *testing.T, expected, actual ChunkData) { + require.Equal(t, expected.meta.MinTime, actual.meta.MinTime, "MinTime mismatch") + require.Equal(t, expected.meta.MaxTime, actual.meta.MaxTime, "MaxTime mismatch") + require.Equal(t, expected.labels, actual.labels, "Labels mismatch") +} + +func generateTestChunkData(totalEntries int) []struct { + labels labels.Labels + entries []*logproto.Entry +} { + var chunkData []struct { + labels labels.Labels + entries []*logproto.Entry + } + + entriesPerChunk := defaultBatchSize * 2 // Each chunk will contain 2 batches worth of entries + numChunks := (totalEntries + entriesPerChunk - 1) / entriesPerChunk + + for i := 0; i < numChunks; i++ { + startIndex := i * entriesPerChunk + endIndex := (i + 1) * entriesPerChunk + if endIndex > totalEntries { + endIndex = totalEntries + } + + chunkData = append(chunkData, struct { + labels labels.Labels + entries []*logproto.Entry + }{ + labels: labels.FromStrings("app", fmt.Sprintf("test%d", i)), + entries: generateEntries(startIndex, endIndex-1), + }) + } + + return chunkData +} + +func writeChunksToStorage(t *testing.T, storage *mockBlockStorage, chunkData []struct { + labels labels.Labels + entries []*logproto.Entry +}, +) []ChunkData { + chks := make([]ChunkData, 0, len(chunkData)) + for i, cd := range chunkData { + var buf bytes.Buffer + chunkID := fmt.Sprintf("chunk%d", i) + _, err := walchunks.WriteChunk(&buf, cd.entries, walchunks.EncodingSnappy) + require.NoError(t, err) + + storage.data[wal.Dir+chunkID] = buf.Bytes() + chks = append(chks, newChunkData(chunkID, labelsToScratchBuilder(cd.labels), &walchunks.Meta{ + Ref: walchunks.NewChunkRef(0, uint64(buf.Len())), + MinTime: cd.entries[0].Timestamp.UnixNano(), + MaxTime: cd.entries[len(cd.entries)-1].Timestamp.UnixNano(), + })) + } + return chks +} + +func generateEntries(start, end int) []*logproto.Entry { + var entries []*logproto.Entry + for i := start; i <= end; i++ { + entries = append(entries, &logproto.Entry{ + Timestamp: time.Unix(int64(i), 0), + Line: fmt.Sprintf("line%d", i), + }) + } + return entries +} + +func flattenEntries(chunkData []struct { + labels labels.Labels + entries []*logproto.Entry +}, +) []logproto.Entry { + var result []logproto.Entry + for _, cd := range chunkData { + for _, e := range cd.entries { + result = append(result, logproto.Entry{Timestamp: e.Timestamp, Line: e.Line}) + } + } + return result +} + +func reverseEntries(entries []logproto.Entry) []logproto.Entry { + for i := 0; i < len(entries)/2; i++ { + j := len(entries) - 1 - i + entries[i], entries[j] = entries[j], entries[i] + } + return entries +} + +func selectEntries(entries []logproto.Entry, start, end int) []logproto.Entry { + var result []logproto.Entry + for _, e := range entries { + if e.Timestamp.Unix() >= int64(start) && e.Timestamp.Unix() < int64(end) { + result = append(result, e) + } + } + return result +} + +func entriesToSamples(entries []logproto.Entry) []logproto.Sample { + var samples []logproto.Sample + for _, e := range entries { + samples = append(samples, logproto.Sample{ + Timestamp: e.Timestamp.UnixNano(), + Value: float64(1), // Use timestamp as value for simplicity + }) + } + return samples +} + +func iterateEntries(iterator *ChunksEntryIterator[iter.EntryIterator]) []logproto.Entry { + var result []logproto.Entry + for iterator.Next() { + entry := iterator.At() + result = append(result, logproto.Entry{Timestamp: entry.Timestamp, Line: entry.Line}) + } + return result +} + +func iterateSamples(iterator *ChunksSampleIterator[iter.SampleIterator]) []logproto.Sample { + var result []logproto.Sample + for iterator.Next() { + result = append(result, iterator.At()) + } + return result +} + +func assertEqualEntries(t *testing.T, expected, actual []logproto.Entry) { + require.Equal(t, len(expected), len(actual), "Number of entries mismatch") + for i := range expected { + require.Equal(t, expected[i].Timestamp, actual[i].Timestamp, "Timestamp mismatch at index %d", i) + require.Equal(t, expected[i].Line, actual[i].Line, "Line mismatch at index %d", i) + } +} + +func assertEqualSamples(t *testing.T, expected, actual []logproto.Sample) { + require.Equal(t, len(expected), len(actual), "Number of samples mismatch") + for i := range expected { + require.Equal(t, expected[i].Timestamp, actual[i].Timestamp, "Timestamp mismatch at index %d", i) + require.Equal(t, expected[i].Value, actual[i].Value, "Value mismatch at index %d", i) + } +} + +func labelsToScratchBuilder(lbs labels.Labels) *labels.ScratchBuilder { + sb := labels.NewScratchBuilder(len(lbs)) + sb.Reset() + for i := 0; i < len(lbs); i++ { + sb.Add(lbs[i].Name, lbs[i].Value) + } + return &sb +} diff --git a/pkg/querier-rf1/wal/querier.go b/pkg/querier-rf1/wal/querier.go new file mode 100644 index 0000000000000..0fb2cc23dc525 --- /dev/null +++ b/pkg/querier-rf1/wal/querier.go @@ -0,0 +1,203 @@ +package wal + +import ( + "bytes" + "context" + "io" + "sync" + + "github.com/opentracing/opentracing-go" + "github.com/prometheus/prometheus/model/labels" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + + "github.com/grafana/dskit/tenant" + + "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" + "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/logql" + "github.com/grafana/loki/v3/pkg/storage/wal" + "github.com/grafana/loki/v3/pkg/storage/wal/chunks" + "github.com/grafana/loki/v3/pkg/storage/wal/index" +) + +var _ logql.Querier = (*Querier)(nil) + +type BlockStorage interface { + GetObjectRange(ctx context.Context, objectKey string, off, length int64) (io.ReadCloser, error) +} + +type Metastore interface { + ListBlocksForQuery(ctx context.Context, in *metastorepb.ListBlocksForQueryRequest, opts ...grpc.CallOption) (*metastorepb.ListBlocksForQueryResponse, error) +} + +type Querier struct { + blockStorage BlockStorage + metaStore Metastore +} + +func New( + metaStore Metastore, + blockStorage BlockStorage, +) (*Querier, error) { + return &Querier{ + blockStorage: blockStorage, + metaStore: metaStore, + }, nil +} + +func (q *Querier) SelectLogs(ctx context.Context, req logql.SelectLogParams) (iter.EntryIterator, error) { + // todo request validation and delete markers. + tenantID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + expr, err := req.LogSelector() + if err != nil { + return nil, err + } + matchers := expr.Matchers() + // todo: not sure if Pipeline is thread safe + pipeline, err := expr.Pipeline() + if err != nil { + return nil, err + } + + chks, err := q.matchingChunks(ctx, tenantID, req.Start.UnixNano(), req.End.UnixNano(), matchers...) + if err != nil { + return nil, err + } + + return NewChunksEntryIterator(ctx, + q.blockStorage, + chks, + pipeline, + req.Direction, + req.Start.UnixNano(), + req.End.UnixNano()), nil +} + +func (q *Querier) SelectSamples(ctx context.Context, req logql.SelectSampleParams) (iter.SampleIterator, error) { + // todo request validation and delete markers. + tenantID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + expr, err := req.Expr() + if err != nil { + return nil, err + } + selector, err := expr.Selector() + if err != nil { + return nil, err + } + matchers := selector.Matchers() + // todo: not sure if Extractor is thread safe + + extractor, err := expr.Extractor() + if err != nil { + return nil, err + } + + chks, err := q.matchingChunks(ctx, tenantID, req.Start.UnixNano(), req.End.UnixNano(), matchers...) + if err != nil { + return nil, err + } + + return NewChunksSampleIterator(ctx, + q.blockStorage, + chks, + extractor, + req.Start.UnixNano(), + req.End.UnixNano()), nil +} + +func (q *Querier) matchingChunks(ctx context.Context, tenantID string, from, through int64, matchers ...*labels.Matcher) ([]ChunkData, error) { + sp, ctx := opentracing.StartSpanFromContext(ctx, "matchingChunks") + defer sp.Finish() + // todo support sharding + var ( + lazyChunks []ChunkData + mtx sync.Mutex + ) + + err := q.forSeries(ctx, &metastorepb.ListBlocksForQueryRequest{ + TenantId: tenantID, + StartTime: from, + EndTime: through, + }, func(id string, lbs *labels.ScratchBuilder, chk *chunks.Meta) error { + mtx.Lock() + lazyChunks = append(lazyChunks, newChunkData(id, lbs, chk)) + mtx.Unlock() + return nil + }, matchers...) + if err != nil { + return nil, err + } + if sp != nil { + sp.LogKV("matchedChunks", len(lazyChunks)) + } + return lazyChunks, nil +} + +func (q *Querier) forSeries(ctx context.Context, req *metastorepb.ListBlocksForQueryRequest, fn func(string, *labels.ScratchBuilder, *chunks.Meta) error, matchers ...*labels.Matcher) error { + // copy matchers to avoid modifying the original slice. + ms := make([]*labels.Matcher, 0, len(matchers)+1) + ms = append(ms, matchers...) + ms = append(ms, labels.MustNewMatcher(labels.MatchEqual, index.TenantLabel, req.TenantId)) + + return q.forIndices(ctx, req, func(ir *index.Reader, id string) error { + bufLbls := labels.ScratchBuilder{} + chunks := make([]chunks.Meta, 0, 1) + p, err := ir.PostingsForMatchers(ctx, ms...) + if err != nil { + return err + } + for p.Next() { + err := ir.Series(p.At(), &bufLbls, &chunks) + if err != nil { + return err + } + if err := fn(id, &bufLbls, &chunks[0]); err != nil { + return err + } + } + return p.Err() + }) +} + +func (q *Querier) forIndices(ctx context.Context, req *metastorepb.ListBlocksForQueryRequest, fn func(ir *index.Reader, id string) error) error { + resp, err := q.metaStore.ListBlocksForQuery(ctx, req) + if err != nil { + return err + } + metas := resp.Blocks + if len(metas) == 0 { + return nil + } + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(32) + for _, meta := range metas { + + meta := meta + g.Go(func() error { + reader, err := q.blockStorage.GetObjectRange(ctx, wal.Dir+meta.Id, meta.IndexRef.Offset, meta.IndexRef.Length) + if err != nil { + return err + } + defer reader.Close() + // todo: use a buffer pool + buf := bytes.NewBuffer(make([]byte, 0, meta.IndexRef.Length)) + _, err = buf.ReadFrom(reader) + if err != nil { + return err + } + index, err := index.NewReader(index.RealByteSlice(buf.Bytes())) + if err != nil { + return err + } + return fn(index, meta.Id) + }) + } + return g.Wait() +} diff --git a/pkg/querier-rf1/wal/querier_test.go b/pkg/querier-rf1/wal/querier_test.go new file mode 100644 index 0000000000000..5d446b8515902 --- /dev/null +++ b/pkg/querier-rf1/wal/querier_test.go @@ -0,0 +1,697 @@ +package wal + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + "testing" + "time" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/grafana/dskit/user" + + "github.com/grafana/loki/v3/pkg/ingester-rf1/metastore/metastorepb" + "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql" + "github.com/grafana/loki/v3/pkg/logql/syntax" + "github.com/grafana/loki/v3/pkg/querier/plan" + "github.com/grafana/loki/v3/pkg/storage/wal" + "github.com/grafana/loki/v3/pkg/storage/wal/chunks" +) + +// MockStorage is a simple in-memory storage for testing +type MockStorage struct { + data map[string][]byte +} + +func NewMockStorage() *MockStorage { + return &MockStorage{data: make(map[string][]byte)} +} + +func (m *MockStorage) GetObjectRange(_ context.Context, objectKey string, off, length int64) (io.ReadCloser, error) { + data, ok := m.data[objectKey] + if !ok { + return nil, fmt.Errorf("object not found: %s", objectKey) + } + return io.NopCloser(bytes.NewReader(data[off : off+length])), nil +} + +func (m *MockStorage) PutObject(objectKey string, data []byte) { + m.data[objectKey] = data +} + +// MockMetastore is a simple in-memory metastore for testing +type MockMetastore struct { + blocks map[string][]*metastorepb.BlockMeta +} + +func NewMockMetastore() *MockMetastore { + return &MockMetastore{blocks: make(map[string][]*metastorepb.BlockMeta)} +} + +func (m *MockMetastore) ListBlocksForQuery(_ context.Context, req *metastorepb.ListBlocksForQueryRequest, _ ...grpc.CallOption) (*metastorepb.ListBlocksForQueryResponse, error) { + blocks := m.blocks[req.TenantId] + var result []*metastorepb.BlockMeta + for _, block := range blocks { + if block.MinTime <= req.EndTime && block.MaxTime >= req.StartTime { + result = append(result, block) + } + } + return &metastorepb.ListBlocksForQueryResponse{Blocks: result}, nil +} + +func (m *MockMetastore) AddBlock(tenantID string, block *metastorepb.BlockMeta) { + m.blocks[tenantID] = append(m.blocks[tenantID], block) +} + +func TestQuerier_SelectLogs(t *testing.T) { + storage := NewMockStorage() + metastore := NewMockMetastore() + + querier, err := New(metastore, storage) + require.NoError(t, err) + + tenantID := "test-tenant" + ctx := user.InjectOrgID(context.Background(), tenantID) + + // Create expanded test data + testData := []struct { + labels labels.Labels + entries []*logproto.Entry + }{ + { + labels: labels.FromStrings("app", "test1", "env", "prod"), + entries: generateEntries(1000, 1050), + }, + { + labels: labels.FromStrings("app", "test2", "env", "staging"), + entries: generateEntries(1025, 1075), + }, + { + labels: labels.FromStrings("app", "test3", "env", "dev", "version", "v1"), + entries: generateEntries(1050, 1100), + }, + { + labels: labels.FromStrings("app", "test4", "env", "prod", "version", "v2"), + entries: generateEntries(1075, 1125), + }, + } + + // Setup test data + setupTestData(t, storage, metastore, tenantID, testData) + + // Test cases + testCases := []struct { + name string + query string + expectedCount int + expectedFirst logproto.Entry + expectedLast logproto.Entry + }{ + { + name: "Query all logs", + query: `{app=~"test.*"}`, + expectedCount: 204, + expectedFirst: logproto.Entry{ + Timestamp: time.Unix(1000, 0), + Line: "line1000", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test1"}, + {Name: "env", Value: "prod"}, + }, + }, + expectedLast: logproto.Entry{ + Timestamp: time.Unix(1125, 0), + Line: "line1125", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test4"}, + {Name: "env", Value: "prod"}, + {Name: "version", Value: "v2"}, + }, + }, + }, + { + name: "Query specific app", + query: `{app="test1"}`, + expectedCount: 51, + expectedFirst: logproto.Entry{ + Timestamp: time.Unix(1000, 0), + Line: "line1000", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test1"}, + {Name: "env", Value: "prod"}, + }, + }, + expectedLast: logproto.Entry{ + Timestamp: time.Unix(1050, 0), + Line: "line1050", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test1"}, + {Name: "env", Value: "prod"}, + }, + }, + }, + { + name: "Query with multiple label equality", + query: `{app="test4", env="prod"}`, + expectedCount: 51, + expectedFirst: logproto.Entry{ + Timestamp: time.Unix(1075, 0), + Line: "line1075", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test4"}, + {Name: "env", Value: "prod"}, + {Name: "version", Value: "v2"}, + }, + }, + expectedLast: logproto.Entry{ + Timestamp: time.Unix(1125, 0), + Line: "line1125", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test4"}, + {Name: "env", Value: "prod"}, + {Name: "version", Value: "v2"}, + }, + }, + }, + { + name: "Query with negative regex", + query: `{app=~"test.*", env!~"stag.*|dev"}`, + expectedCount: 102, + expectedFirst: logproto.Entry{ + Timestamp: time.Unix(1000, 0), + Line: "line1000", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test1"}, + {Name: "env", Value: "prod"}, + }, + }, + expectedLast: logproto.Entry{ + Timestamp: time.Unix(1125, 0), + Line: "line1125", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test4"}, + {Name: "env", Value: "prod"}, + {Name: "version", Value: "v2"}, + }, + }, + }, + { + name: "Query with label presence", + query: `{app=~"test.*", version=""}`, + expectedCount: 102, + expectedFirst: logproto.Entry{ + Timestamp: time.Unix(1000, 0), + Line: "line1000", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test1"}, + {Name: "env", Value: "prod"}, + }, + }, + expectedLast: logproto.Entry{ + Timestamp: time.Unix(1075, 0), + Line: "line1075", + Parsed: []logproto.LabelAdapter{ + {Name: "app", Value: "test2"}, + {Name: "env", Value: "staging"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + expr, err := syntax.ParseExpr(tc.query) + require.NoError(t, err) + + req := logql.SelectLogParams{ + QueryRequest: &logproto.QueryRequest{ + Selector: tc.query, + Start: time.Unix(1000, 0), + End: time.Unix(1126, 0), + Limit: 10000, + Direction: logproto.FORWARD, + Plan: &plan.QueryPlan{ + AST: expr, + }, + }, + } + + iter, err := querier.SelectLogs(ctx, req) + require.NoError(t, err) + + results := collectPushEntries(t, iter) + + assert.Len(t, results, tc.expectedCount, "Unexpected number of log entries") + if len(results) > 0 { + assert.Equal(t, tc.expectedFirst, results[0], "First log entry mismatch") + assert.Equal(t, tc.expectedLast, results[len(results)-1], "Last log entry mismatch") + } + }) + } +} + +// SampleWithLabels is a new struct to hold both the sample and its labels +type SampleWithLabels struct { + Sample logproto.Sample + Labels labels.Labels +} + +func TestQuerier_SelectSamples(t *testing.T) { + storage := NewMockStorage() + metastore := NewMockMetastore() + + querier, err := New(metastore, storage) + require.NoError(t, err) + + tenantID := "test-tenant" + ctx := user.InjectOrgID(context.Background(), tenantID) + + // Create test data + testData := []struct { + labels labels.Labels + samples []logproto.Sample + }{ + { + labels: labels.FromStrings("app", "test1", "env", "prod"), + samples: generateSamples(1000, 1050, 1), + }, + { + labels: labels.FromStrings("app", "test2", "env", "staging"), + samples: generateSamples(1025, 1075, 2), + }, + { + labels: labels.FromStrings("app", "test3", "env", "dev", "version", "v1"), + samples: generateSamples(1050, 1100, 3), + }, + { + labels: labels.FromStrings("app", "test4", "env", "prod", "version", "v2"), + samples: generateSamples(1075, 1125, 4), + }, + } + + // Setup test data + setupTestSampleData(t, storage, metastore, tenantID, testData) + + // Test cases + testCases := []struct { + name string + query string + expectedCount int + expectedFirst SampleWithLabels + expectedLast SampleWithLabels + }{ + { + name: "Query all samples", + query: `sum_over_time({app=~"test.*"} | label_format v="{{__line__}}" | unwrap v[1s])`, + expectedCount: 204, + expectedFirst: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1000, 0).UnixNano(), + Value: 1, + }, + Labels: labels.FromStrings("app", "test1", "env", "prod"), + }, + expectedLast: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1125, 0).UnixNano(), + Value: 4, + }, + Labels: labels.FromStrings("app", "test4", "env", "prod", "version", "v2"), + }, + }, + { + name: "Query specific app", + query: `sum_over_time({app="test1"}| label_format v="{{__line__}}" | unwrap v[1s])`, + expectedCount: 51, + expectedFirst: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1000, 0).UnixNano(), + Value: 1, + }, + Labels: labels.FromStrings("app", "test1", "env", "prod"), + }, + expectedLast: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1050, 0).UnixNano(), + Value: 1, + }, + Labels: labels.FromStrings("app", "test1", "env", "prod"), + }, + }, + { + name: "Query with multiple label equality", + query: `sum_over_time({app="test4", env="prod"}| label_format v="{{__line__}}" | unwrap v[1s])`, + expectedCount: 51, + expectedFirst: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1075, 0).UnixNano(), + Value: 4, + }, + Labels: labels.FromStrings("app", "test4", "env", "prod", "version", "v2"), + }, + expectedLast: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1125, 0).UnixNano(), + Value: 4, + }, + Labels: labels.FromStrings("app", "test4", "env", "prod", "version", "v2"), + }, + }, + { + name: "Query with negative regex", + query: `sum_over_time({app=~"test.*", env!~"stag.*|dev"}| label_format v="{{__line__}}" | unwrap v[1s])`, + expectedCount: 102, + expectedFirst: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1000, 0).UnixNano(), + Value: 1, + }, + Labels: labels.FromStrings("app", "test1", "env", "prod"), + }, + expectedLast: SampleWithLabels{ + Sample: logproto.Sample{ + Timestamp: time.Unix(1125, 0).UnixNano(), + Value: 4, + }, + Labels: labels.FromStrings("app", "test4", "env", "prod", "version", "v2"), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + expr, err := syntax.ParseExpr(tc.query) + require.NoError(t, err) + + req := logql.SelectSampleParams{ + SampleQueryRequest: &logproto.SampleQueryRequest{ + Selector: tc.query, + Start: time.Unix(1000, 0), + End: time.Unix(1126, 0), + Plan: &plan.QueryPlan{ + AST: expr, + }, + }, + } + + iter, err := querier.SelectSamples(ctx, req) + require.NoError(t, err) + + results := collectSamplesWithLabels(t, iter) + + assert.Len(t, results, tc.expectedCount, "Unexpected number of samples") + if len(results) > 0 { + assert.Equal(t, tc.expectedFirst.Sample, results[0].Sample, "First sample mismatch") + assert.Equal(t, tc.expectedFirst.Labels, results[0].Labels, "First sample labels mismatch") + assert.Equal(t, tc.expectedLast.Sample, results[len(results)-1].Sample, "Last sample mismatch") + assert.Equal(t, tc.expectedLast.Labels, results[len(results)-1].Labels, "Last sample labels mismatch") + } + }) + } +} + +func TestQuerier_matchingChunks(t *testing.T) { + storage := NewMockStorage() + metastore := NewMockMetastore() + + querier, err := New(metastore, storage) + require.NoError(t, err) + + tenantID := "test-tenant" + ctx := user.InjectOrgID(context.Background(), tenantID) + + // Create test data + testData := []struct { + labels labels.Labels + entries []*logproto.Entry + }{ + { + labels: labels.FromStrings("app", "app1", "env", "prod"), + entries: generateEntries(1000, 1050), + }, + { + labels: labels.FromStrings("app", "app2", "env", "staging"), + entries: generateEntries(1025, 1075), + }, + { + labels: labels.FromStrings("app", "app3", "env", "dev"), + entries: generateEntries(1050, 1100), + }, + } + + // Setup test data + setupTestData(t, storage, metastore, tenantID, testData) + + // Test cases + testCases := []struct { + name string + matchers []*labels.Matcher + start int64 + end int64 + expectedChunks []ChunkData + }{ + { + name: "Equality matcher", + matchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchEqual, "app", "app1"), + }, + start: time.Unix(1000, 0).UnixNano(), + end: time.Unix(1100, 0).UnixNano(), + expectedChunks: []ChunkData{ + { + labels: labels.FromStrings("app", "app1", "env", "prod"), + meta: &chunks.Meta{MinTime: time.Unix(1000, 0).UnixNano(), MaxTime: time.Unix(1050, 0).UnixNano()}, + }, + }, + }, + { + name: "Negative matcher", + matchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchNotEqual, "app", "app1"), + }, + start: time.Unix(1000, 0).UnixNano(), + end: time.Unix(1100, 0).UnixNano(), + expectedChunks: []ChunkData{ + { + labels: labels.FromStrings("app", "app2", "env", "staging"), + meta: &chunks.Meta{MinTime: time.Unix(1025, 0).UnixNano(), MaxTime: time.Unix(1075, 0).UnixNano()}, + }, + { + labels: labels.FromStrings("app", "app3", "env", "dev"), + meta: &chunks.Meta{MinTime: time.Unix(1050, 0).UnixNano(), MaxTime: time.Unix(1100, 0).UnixNano()}, + }, + }, + }, + { + name: "Regex matcher", + matchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, "app", "app[12]"), + }, + start: time.Unix(1000, 0).UnixNano(), + end: time.Unix(1100, 0).UnixNano(), + expectedChunks: []ChunkData{ + { + labels: labels.FromStrings("app", "app1", "env", "prod"), + meta: &chunks.Meta{MinTime: time.Unix(1000, 0).UnixNano(), MaxTime: time.Unix(1050, 0).UnixNano()}, + }, + { + labels: labels.FromStrings("app", "app2", "env", "staging"), + meta: &chunks.Meta{MinTime: time.Unix(1025, 0).UnixNano(), MaxTime: time.Unix(1075, 0).UnixNano()}, + }, + }, + }, + { + name: "Not regex matcher", + matchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchNotRegexp, "app", "app[12]"), + }, + start: time.Unix(1000, 0).UnixNano(), + end: time.Unix(1100, 0).UnixNano(), + expectedChunks: []ChunkData{ + { + labels: labels.FromStrings("app", "app3", "env", "dev"), + meta: &chunks.Meta{MinTime: time.Unix(1050, 0).UnixNano(), MaxTime: time.Unix(1100, 0).UnixNano()}, + }, + }, + }, + { + name: "Multiple matchers", + matchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, "app", "app.*"), + labels.MustNewMatcher(labels.MatchNotEqual, "env", "prod"), + }, + start: time.Unix(1000, 0).UnixNano(), + end: time.Unix(1100, 0).UnixNano(), + expectedChunks: []ChunkData{ + { + labels: labels.FromStrings("app", "app2", "env", "staging"), + meta: &chunks.Meta{MinTime: time.Unix(1025, 0).UnixNano(), MaxTime: time.Unix(1075, 0).UnixNano()}, + }, + { + labels: labels.FromStrings("app", "app3", "env", "dev"), + meta: &chunks.Meta{MinTime: time.Unix(1050, 0).UnixNano(), MaxTime: time.Unix(1100, 0).UnixNano()}, + }, + }, + }, + { + name: "Time range filter", + matchers: []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, "app", "app.*"), + }, + start: time.Unix(1080, 0).UnixNano(), + end: time.Unix(1100, 0).UnixNano(), + expectedChunks: []ChunkData{ + { + labels: labels.FromStrings("app", "app3", "env", "dev"), + meta: &chunks.Meta{MinTime: time.Unix(1050, 0).UnixNano(), MaxTime: time.Unix(1100, 0).UnixNano()}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + chunks, err := querier.matchingChunks(ctx, tenantID, tc.start, tc.end, tc.matchers...) + require.NoError(t, err) + + sort.Slice(tc.expectedChunks, func(i, j int) bool { + return tc.expectedChunks[i].labels.String() < tc.expectedChunks[j].labels.String() + }) + sort.Slice(chunks, func(i, j int) bool { + return chunks[i].labels.String() < chunks[j].labels.String() + }) + assert.Equal(t, len(tc.expectedChunks), len(chunks), "Unexpected number of matching chunks") + + // Verify that all returned chunks match the expected chunks + for i, expectedChunk := range tc.expectedChunks { + if i < len(chunks) { + assert.Equal(t, expectedChunk.labels, chunks[i].labels, "Labels mismatch for chunk %d", i) + assert.Equal(t, expectedChunk.meta.MinTime, chunks[i].meta.MinTime, "MinTime mismatch for chunk %d", i) + assert.Equal(t, expectedChunk.meta.MaxTime, chunks[i].meta.MaxTime, "MaxTime mismatch for chunk %d", i) + } + } + + // Additional checks for time range and matchers + for _, chunk := range chunks { + for _, matcher := range tc.matchers { + assert.True(t, matcher.Matches(chunk.labels.Get(matcher.Name)), + "Chunk labels %v do not match criteria %v", chunk.labels, matcher) + } + } + }) + } +} + +func setupTestData(t *testing.T, storage *MockStorage, metastore *MockMetastore, tenantID string, testData []struct { + labels labels.Labels + entries []*logproto.Entry +}, +) { + total := 0 + for i, data := range testData { + segmentID := fmt.Sprintf("segment%d", i) + writer, err := wal.NewWalSegmentWriter() + require.NoError(t, err) + total += len(data.entries) + writer.Append(tenantID, data.labels.String(), data.labels, data.entries, time.Now()) + + var buf bytes.Buffer + _, err = writer.WriteTo(&buf) + require.NoError(t, err) + + segmentData := buf.Bytes() + storage.PutObject(wal.Dir+segmentID, segmentData) + + blockMeta := writer.Meta(segmentID) + metastore.AddBlock(tenantID, blockMeta) + } + t.Log("Total entries in storage:", total) +} + +func collectPushEntries(t *testing.T, iter iter.EntryIterator) []logproto.Entry { + var results []logproto.Entry + for iter.Next() { + entry := iter.At() + lbs := iter.Labels() + parsed, err := syntax.ParseLabels(lbs) + require.NoError(t, err) + results = append(results, logproto.Entry{ + Timestamp: entry.Timestamp, + Line: entry.Line, + Parsed: logproto.FromLabelsToLabelAdapters(parsed), + }) + } + require.NoError(t, iter.Close()) + return results +} + +func collectSamplesWithLabels(t *testing.T, iter iter.SampleIterator) []SampleWithLabels { + var results []SampleWithLabels + for iter.Next() { + sample := iter.At() + labelString := iter.Labels() + parsedLabels, err := syntax.ParseLabels(labelString) + require.NoError(t, err) + results = append(results, SampleWithLabels{ + Sample: sample, + Labels: parsedLabels, + }) + } + require.NoError(t, iter.Close()) + return results +} + +func generateSamples(start, end int64, value float64) []logproto.Sample { + var samples []logproto.Sample + for i := start; i <= end; i++ { + samples = append(samples, logproto.Sample{ + Timestamp: time.Unix(i, 0).UnixNano(), + Value: value, + }) + } + return samples +} + +func setupTestSampleData(t *testing.T, storage *MockStorage, metastore *MockMetastore, tenantID string, testData []struct { + labels labels.Labels + samples []logproto.Sample +}, +) { + total := 0 + for i, data := range testData { + segmentID := fmt.Sprintf("segment%d", i) + writer, err := wal.NewWalSegmentWriter() + require.NoError(t, err) + total += len(data.samples) + + // Convert samples to entries for the WAL writer + entries := make([]*logproto.Entry, len(data.samples)) + for i, sample := range data.samples { + entries[i] = &logproto.Entry{ + Timestamp: time.Unix(0, sample.Timestamp), + Line: fmt.Sprintf("%f", sample.Value), + } + } + + writer.Append(tenantID, data.labels.String(), data.labels, entries, time.Now()) + + var buf bytes.Buffer + _, err = writer.WriteTo(&buf) + require.NoError(t, err) + + segmentData := buf.Bytes() + storage.PutObject(wal.Dir+segmentID, segmentData) + + blockMeta := writer.Meta(segmentID) + metastore.AddBlock(tenantID, blockMeta) + } + t.Log("Total samples in storage:", total) +} diff --git a/pkg/querier/queryrange/roundtrip.go b/pkg/querier/queryrange/roundtrip.go index 3e658f18c86f4..21961d279ae61 100644 --- a/pkg/querier/queryrange/roundtrip.go +++ b/pkg/querier/queryrange/roundtrip.go @@ -28,6 +28,7 @@ import ( "github.com/grafana/loki/v3/pkg/storage/config" "github.com/grafana/loki/v3/pkg/util" "github.com/grafana/loki/v3/pkg/util/constants" + "github.com/grafana/loki/v3/pkg/util/httpreq" logutil "github.com/grafana/loki/v3/pkg/util/log" "github.com/grafana/loki/v3/pkg/util/validation" ) @@ -400,8 +401,9 @@ func (r roundTripper) Do(ctx context.Context, req base.Request) (base.Response, return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } - // Only filter expressions are query sharded - if !e.HasFilter() { + // Some queries we don't want to parallelize as aggressively, like limited queries and `datasample` queries + tags := httpreq.ExtractQueryTagsFromContext(ctx) + if !e.HasFilter() || strings.Contains(tags, "datasample") { return r.limited.Do(ctx, req) } return r.log.Do(ctx, req) diff --git a/pkg/ruler/registry.go b/pkg/ruler/registry.go index 868b7f29a6f94..29297fcab5a74 100644 --- a/pkg/ruler/registry.go +++ b/pkg/ruler/registry.go @@ -128,8 +128,12 @@ func (r *walRegistry) get(tenant string) storage.Storage { } func (r *walRegistry) Appender(ctx context.Context) storage.Appender { + // concurrency-safe retrieval of remote-write config for this tenant, using the global remote-write for defaults + r.overridesMu.Lock() tenant, _ := user.ExtractOrgID(ctx) rwCfg, err := r.getTenantRemoteWriteConfig(tenant, r.config.RemoteWrite) + r.overridesMu.Unlock() + if err != nil { level.Error(r.logger).Log("msg", "error retrieving remote-write config; discarding samples", "user", tenant, "err", err) return discardingAppender{} diff --git a/pkg/ruler/registry_test.go b/pkg/ruler/registry_test.go index 261b6d3836763..7ab12d8962ae6 100644 --- a/pkg/ruler/registry_test.go +++ b/pkg/ruler/registry_test.go @@ -405,6 +405,30 @@ func TestTenantRemoteWriteConfigWithOverrideConcurrentAccess(t *testing.T) { }) } +func TestAppenderConcurrentAccess(t *testing.T) { + require.NotPanics(t, func() { + reg := setupRegistry(t, cfg, newFakeLimits()) + var wg sync.WaitGroup + for i := 0; i < 1000; i++ { + wg.Add(1) + go func(reg *walRegistry) { + defer wg.Done() + + _ = reg.Appender(user.InjectOrgID(context.Background(), enabledRWTenant)) + }(reg) + + wg.Add(1) + go func(reg *walRegistry) { + defer wg.Done() + + _ = reg.Appender(user.InjectOrgID(context.Background(), additionalHeadersRWTenant)) + }(reg) + } + + wg.Wait() + }) +} + func TestTenantRemoteWriteConfigWithoutOverride(t *testing.T) { reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat()) diff --git a/pkg/storage/bloom/v1/builder.go b/pkg/storage/bloom/v1/builder.go index 09a0dc2778f42..56882c4cb140a 100644 --- a/pkg/storage/bloom/v1/builder.go +++ b/pkg/storage/bloom/v1/builder.go @@ -217,6 +217,7 @@ func (mb *MergeBuilder) processNextSeries( ) ( *SeriesWithBlooms, // nextInBlocks pointer update int, // bytes added + int, // chunks added bool, // blocksFinished update bool, // done building block error, // error @@ -230,7 +231,7 @@ func (mb *MergeBuilder) processNextSeries( }() if !mb.store.Next() { - return nil, 0, false, true, nil + return nil, 0, 0, false, true, nil } nextInStore := mb.store.At() @@ -249,7 +250,7 @@ func (mb *MergeBuilder) processNextSeries( } if err := mb.blocks.Err(); err != nil { - return nil, 0, false, false, errors.Wrap(err, "iterating blocks") + return nil, 0, 0, false, false, errors.Wrap(err, "iterating blocks") } blockSeriesIterated++ nextInBlocks = mb.blocks.At() @@ -276,11 +277,11 @@ func (mb *MergeBuilder) processNextSeries( for bloom := range ch { if bloom.Err != nil { - return nil, bytesAdded, false, false, errors.Wrap(bloom.Err, "populating bloom") + return nil, bytesAdded, 0, false, false, errors.Wrap(bloom.Err, "populating bloom") } offset, err := builder.AddBloom(bloom.Bloom) if err != nil { - return nil, bytesAdded, false, false, errors.Wrapf( + return nil, bytesAdded, 0, false, false, errors.Wrapf( err, "adding bloom to block for fp (%s)", nextInStore.Fingerprint, ) } @@ -290,25 +291,29 @@ func (mb *MergeBuilder) processNextSeries( done, err := builder.AddSeries(*nextInStore, offsets) if err != nil { - return nil, bytesAdded, false, false, errors.Wrap(err, "committing series") + return nil, bytesAdded, 0, false, false, errors.Wrap(err, "committing series") } - return nextInBlocks, bytesAdded, blocksFinished, done, nil + return nextInBlocks, bytesAdded, chunksIndexed + chunksCopied, blocksFinished, done, nil } func (mb *MergeBuilder) Build(builder *BlockBuilder) (checksum uint32, totalBytes int, err error) { var ( - nextInBlocks *SeriesWithBlooms - blocksFinished bool // whether any previous blocks have been exhausted while building new block - done bool + nextInBlocks *SeriesWithBlooms + blocksFinished bool // whether any previous blocks have been exhausted while building new block + done bool + totalSeriesAdded = 0 + totalChunksAdded int ) for { - var bytesAdded int - nextInBlocks, bytesAdded, blocksFinished, done, err = mb.processNextSeries(builder, nextInBlocks, blocksFinished) + var bytesAdded, chunksAdded int + nextInBlocks, bytesAdded, chunksAdded, blocksFinished, done, err = mb.processNextSeries(builder, nextInBlocks, blocksFinished) totalBytes += bytesAdded + totalChunksAdded += chunksAdded if err != nil { return 0, totalBytes, errors.Wrap(err, "processing next series") } + totalSeriesAdded++ if done { break } @@ -324,6 +329,8 @@ func (mb *MergeBuilder) Build(builder *BlockBuilder) (checksum uint32, totalByte flushedFor = blockFlushReasonFull } mb.metrics.blockSize.Observe(float64(sz)) + mb.metrics.seriesPerBlock.Observe(float64(totalSeriesAdded)) + mb.metrics.chunksPerBlock.Observe(float64(totalChunksAdded)) mb.metrics.blockFlushReason.WithLabelValues(flushedFor).Inc() checksum, err = builder.Close() diff --git a/pkg/storage/bloom/v1/metrics.go b/pkg/storage/bloom/v1/metrics.go index cb94373185d8e..e2ce99a4702d1 100644 --- a/pkg/storage/bloom/v1/metrics.go +++ b/pkg/storage/bloom/v1/metrics.go @@ -20,6 +20,8 @@ type Metrics struct { insertsTotal *prometheus.CounterVec sourceBytesAdded prometheus.Counter blockSize prometheus.Histogram + seriesPerBlock prometheus.Histogram + chunksPerBlock prometheus.Histogram blockFlushReason *prometheus.CounterVec // reads @@ -120,6 +122,18 @@ func NewMetrics(r prometheus.Registerer) *Metrics { Help: "Size of the bloom block in bytes", Buckets: prometheus.ExponentialBucketsRange(1<<20, 1<<30, 8), }), + seriesPerBlock: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ + Namespace: constants.Loki, + Name: "bloom_series_per_block", + Help: "Number of series per block", + Buckets: prometheus.ExponentialBuckets(1, 2, 9), // 2 --> 256 + }), + chunksPerBlock: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ + Namespace: constants.Loki, + Name: "bloom_chunks_per_block", + Help: "Number of chunks per block", + Buckets: prometheus.ExponentialBuckets(1, 2, 15), // 2 --> 16384 + }), blockFlushReason: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "bloom_block_flush_reason_total", diff --git a/pkg/storage/chunk/chunk.go b/pkg/storage/chunk/chunk.go index e807b5fb87798..6f050f8cbd01d 100644 --- a/pkg/storage/chunk/chunk.go +++ b/pkg/storage/chunk/chunk.go @@ -3,6 +3,7 @@ package chunk import ( "bytes" "encoding/binary" + "fmt" "hash/crc32" "reflect" "strconv" @@ -12,6 +13,8 @@ import ( errs "errors" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/golang/snappy" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" @@ -19,6 +22,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/v3/pkg/logproto" + util_log "github.com/grafana/loki/v3/pkg/util/log" ) var ( @@ -27,6 +31,7 @@ var ( ErrMetadataLength = errs.New("chunk metadata wrong length") ErrDataLength = errs.New("chunk data wrong length") ErrSliceOutOfRange = errs.New("chunk can't be sliced out of its data range") + ErrChunkDecode = errs.New("error decoding freshly created chunk") ) var castagnoliTable = crc32.MakeTable(crc32.Castagnoli) @@ -227,11 +232,11 @@ var writerPool = sync.Pool{ // Encode writes the chunk into a buffer, and calculates the checksum. func (c *Chunk) Encode() error { - return c.EncodeTo(nil) + return c.EncodeTo(nil, util_log.Logger) } // EncodeTo is like Encode but you can provide your own buffer to use. -func (c *Chunk) EncodeTo(buf *bytes.Buffer) error { +func (c *Chunk) EncodeTo(buf *bytes.Buffer, log log.Logger) error { if buf == nil { buf = bytes.NewBuffer(nil) } @@ -275,6 +280,32 @@ func (c *Chunk) EncodeTo(buf *bytes.Buffer) error { // Now work out the checksum c.encoded = buf.Bytes() c.Checksum = crc32.Checksum(c.encoded, castagnoliTable) + + newCh := Chunk{ + ChunkRef: logproto.ChunkRef{ + UserID: c.UserID, + Fingerprint: c.Fingerprint, + From: c.From, + Through: c.Through, + Checksum: c.Checksum, + }, + } + + if err := newCh.Decode(NewDecodeContext(), c.encoded); err != nil { + externalKey := fmt.Sprintf( + "%s/%x/%x:%x:%x", + c.UserID, + c.Fingerprint, + int64(c.From), + int64(c.Through), + c.Checksum, + ) + level.Error(log). + Log("msg", "error decoding freshly created chunk", "err", err, "key", externalKey) + + return ErrChunkDecode + } + return nil } diff --git a/pkg/storage/chunk/client/metrics.go b/pkg/storage/chunk/client/metrics.go index 76ca20a1bac5f..dfe789e7af2e3 100644 --- a/pkg/storage/chunk/client/metrics.go +++ b/pkg/storage/chunk/client/metrics.go @@ -29,6 +29,7 @@ type ChunkClientMetrics struct { chunksSizePutPerUser *prometheus.CounterVec chunksFetchedPerUser *prometheus.CounterVec chunksSizeFetchedPerUser *prometheus.CounterVec + chunkDecodeFailures *prometheus.CounterVec } func NewChunkClientMetrics(reg prometheus.Registerer) ChunkClientMetrics { @@ -53,6 +54,11 @@ func NewChunkClientMetrics(reg prometheus.Registerer) ChunkClientMetrics { Name: "chunk_store_fetched_chunk_bytes_total", Help: "Total bytes fetched in chunks per user.", }, []string{"user"}), + chunkDecodeFailures: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "chunk_store_decode_failures_total", + Help: "Total chunk decoding failures.", + }, []string{"user"}), } } @@ -85,6 +91,17 @@ func (c MetricsChunkClient) PutChunks(ctx context.Context, chunks []chunk.Chunk) func (c MetricsChunkClient) GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error) { chks, err := c.Client.GetChunks(ctx, chunks) if err != nil { + // Get chunks fetches chunks in parallel, and returns any error. As a result we don't know which chunk failed, + // so we increment the metric for all tenants with chunks in the request. I think in practice we're only ever + // fetching chunks for a single tenant at a time anyway? + affectedUsers := map[string]struct{}{} + for _, chk := range chks { + affectedUsers[chk.UserID] = struct{}{} + } + for user := range affectedUsers { + c.metrics.chunkDecodeFailures.WithLabelValues(user).Inc() + } + return chks, err } diff --git a/pkg/storage/chunk/client/object_client.go b/pkg/storage/chunk/client/object_client.go index 2f5c3e263e853..225f5025b1d51 100644 --- a/pkg/storage/chunk/client/object_client.go +++ b/pkg/storage/chunk/client/object_client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/base64" + "fmt" "io" "strings" "time" @@ -186,7 +187,14 @@ func (o *client) getChunk(ctx context.Context, decodeContext *chunk.DecodeContex } if err := c.Decode(decodeContext, buf.Bytes()); err != nil { - return chunk.Chunk{}, errors.WithStack(err) + return chunk.Chunk{}, errors.WithStack( + fmt.Errorf( + "failed to decode chunk '%s' for tenant `%s`: %w", + key, + c.ChunkRef.UserID, + err, + ), + ) } return c, nil } diff --git a/pkg/storage/chunk/fetcher/fetcher.go b/pkg/storage/chunk/fetcher/fetcher.go index cf763b9cbedc9..45b6970045a91 100644 --- a/pkg/storage/chunk/fetcher/fetcher.go +++ b/pkg/storage/chunk/fetcher/fetcher.go @@ -9,7 +9,6 @@ import ( "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/prometheus/promql" "github.com/grafana/loki/v3/pkg/logqlmodel/stats" "github.com/grafana/loki/v3/pkg/storage/chunk" @@ -218,8 +217,7 @@ func (c *Fetcher) FetchChunks(ctx context.Context, chunks []chunk.Chunk) ([]chun } if err != nil { - // Don't rely on Cortex error translation here. - return nil, promql.ErrStorage{Err: err} + level.Error(log).Log("msg", "failed downloading chunks", "err", err) } allChunks := append(fromCache, fromStorage...) diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index 5932b08afe8ff..01066d69ad5f6 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -488,6 +488,9 @@ func NewChunkClient(name string, cfg Config, schemaCfg config.SchemaConfig, cc c if err != nil { return nil, err } + if cfg.CongestionControl.Enabled { + c = cc.Wrap(c) + } return client.NewClientWithMaxParallel(c, nil, cfg.MaxParallelGetChunk, schemaCfg), nil case types.StorageTypeGCS: diff --git a/pkg/storage/stores/shipper/indexshipper/tsdb/index/postings.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/postings.go index 248cd523dab59..7c2dd99023b7d 100644 --- a/pkg/storage/stores/shipper/indexshipper/tsdb/index/postings.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/index/postings.go @@ -424,6 +424,13 @@ func EmptyPostings() Postings { return emptyPostings } +// IsEmptyPostingsType returns true if the postings are an empty postings list. +// When this function returns false, it doesn't mean that the postings isn't empty +// (it could be an empty intersection of two non-empty postings, for example). +func IsEmptyPostingsType(p Postings) bool { + return p == emptyPostings +} + // ErrPostings returns new postings that immediately error. func ErrPostings(err error) Postings { return errPostings{err} diff --git a/pkg/storage/wal/chunks/chunks.go b/pkg/storage/wal/chunks/chunks.go index f0f2625596f5d..60b7b6612446b 100644 --- a/pkg/storage/wal/chunks/chunks.go +++ b/pkg/storage/wal/chunks/chunks.go @@ -15,6 +15,7 @@ import ( "github.com/klauspost/compress/s2" "github.com/grafana/loki/v3/pkg/chunkenc" + "github.com/grafana/loki/v3/pkg/iter" "github.com/grafana/loki/v3/pkg/logproto" ) @@ -27,7 +28,10 @@ const ( ) // Initialize the CRC32 table -var castagnoliTable *crc32.Table +var ( + castagnoliTable *crc32.Table + _ iter.EntryIterator = (*entryBufferedIterator)(nil) +) func init() { castagnoliTable = crc32.MakeTable(crc32.Castagnoli) diff --git a/pkg/storage/wal/chunks/doc.go b/pkg/storage/wal/chunks/doc.go new file mode 100644 index 0000000000000..b3b50ed818c68 --- /dev/null +++ b/pkg/storage/wal/chunks/doc.go @@ -0,0 +1,37 @@ +// Package chunks provides functionality for efficient storage and retrieval of log data and metrics. +// +// The chunks package implements a compact and performant way to store and access +// log entries and metric samples. It uses various compression and encoding techniques to minimize +// storage requirements while maintaining fast access times. +// +// Key features: +// - Efficient chunk writing with multiple encoding options +// - Fast chunk reading with iterators for forward and backward traversal +// - Support for time-based filtering of log entries and metric samples +// - Integration with Loki's log query language (LogQL) for advanced filtering and processing +// - Separate iterators for log entries and metric samples +// +// Main types and functions: +// - WriteChunk: Writes log entries to a compressed chunk format +// - NewChunkReader: Creates a reader for parsing and accessing chunk data +// - NewEntryIterator: Provides an iterator for efficient traversal of log entries in a chunk +// - NewSampleIterator: Provides an iterator for efficient traversal of metric samples in a chunk +// +// Entry Iterator: +// The EntryIterator allows efficient traversal of log entries within a chunk. It supports +// both forward and backward iteration, time-based filtering, and integration with LogQL pipelines +// for advanced log processing. +// +// Sample Iterator: +// The SampleIterator enables efficient traversal of metric samples within a chunk. It supports +// time-based filtering and integration with LogQL extractors for advanced metric processing. +// This iterator is particularly useful for handling numeric data extracted from logs or +// pre-aggregated metrics. +// +// Both iterators implement methods for accessing the current entry or sample, checking for errors, +// and retrieving associated labels and stream hashes. +// +// This package is designed to work seamlessly with other components of the Loki +// log aggregation system, providing a crucial layer for data storage and retrieval of +// both logs and metrics. +package chunks diff --git a/pkg/storage/wal/chunks/entry_iterator.go b/pkg/storage/wal/chunks/entry_iterator.go new file mode 100644 index 0000000000000..9a127266b07a8 --- /dev/null +++ b/pkg/storage/wal/chunks/entry_iterator.go @@ -0,0 +1,115 @@ +package chunks + +import ( + "time" + + "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql/log" + + "github.com/grafana/loki/pkg/push" +) + +type entryBufferedIterator struct { + reader *ChunkReader + pipeline log.StreamPipeline + from, through int64 + + cur logproto.Entry + currLabels log.LabelsResult +} + +// NewEntryIterator creates an iterator for efficiently traversing log entries in a chunk. +// It takes compressed chunk data, a processing pipeline, iteration direction, and a time range. +// The returned iterator filters entries based on the time range and applies the given pipeline. +// It handles both forward and backward iteration. +// +// Parameters: +// - chunkData: Compressed chunk data containing log entries +// - pipeline: StreamPipeline for processing and filtering entries +// - direction: Direction of iteration (FORWARD or BACKWARD) +// - from: Start timestamp (inclusive) for filtering entries +// - through: End timestamp (exclusive) for filtering entries +// +// Returns an EntryIterator and an error if creation fails. +func NewEntryIterator( + chunkData []byte, + pipeline log.StreamPipeline, + direction logproto.Direction, + from, through int64, +) (iter.EntryIterator, error) { + chkReader, err := NewChunkReader(chunkData) + if err != nil { + return nil, err + } + it := &entryBufferedIterator{ + reader: chkReader, + pipeline: pipeline, + from: from, + through: through, + } + if direction == logproto.FORWARD { + return it, nil + } + return iter.NewEntryReversedIter(it) +} + +// At implements iter.EntryIterator. +func (e *entryBufferedIterator) At() push.Entry { + return e.cur +} + +// Close implements iter.EntryIterator. +func (e *entryBufferedIterator) Close() error { + return e.reader.Close() +} + +// Err implements iter.EntryIterator. +func (e *entryBufferedIterator) Err() error { + return e.reader.Err() +} + +// Labels implements iter.EntryIterator. +func (e *entryBufferedIterator) Labels() string { + return e.currLabels.String() +} + +// Next implements iter.EntryIterator. +func (e *entryBufferedIterator) Next() bool { + for e.reader.Next() { + ts, line := e.reader.At() + // check if the timestamp is within the range before applying the pipeline. + if ts < e.from { + continue + } + if ts >= e.through { + return false + } + // todo: structured metadata. + newLine, lbs, matches := e.pipeline.Process(ts, line) + if !matches { + continue + } + e.currLabels = lbs + e.cur.Timestamp = time.Unix(0, ts) + e.cur.Line = string(newLine) + e.cur.StructuredMetadata = logproto.FromLabelsToLabelAdapters(lbs.StructuredMetadata()) + e.cur.Parsed = logproto.FromLabelsToLabelAdapters(lbs.Parsed()) + return true + } + return false +} + +// StreamHash implements iter.EntryIterator. +func (e *entryBufferedIterator) StreamHash() uint64 { + return e.pipeline.BaseLabels().Hash() +} + +type sampleBufferedIterator struct { + reader *ChunkReader + pipeline log.StreamSampleExtractor + from, through int64 + + cur logproto.Sample + currLabels log.LabelsResult +} diff --git a/pkg/storage/wal/chunks/entry_iterator_test.go b/pkg/storage/wal/chunks/entry_iterator_test.go new file mode 100644 index 0000000000000..a098161134a55 --- /dev/null +++ b/pkg/storage/wal/chunks/entry_iterator_test.go @@ -0,0 +1,143 @@ +package chunks + +import ( + "bytes" + "testing" + "time" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql/log" + "github.com/grafana/loki/v3/pkg/logql/syntax" +) + +func TestNewEntryIterator(t *testing.T) { + tests := []struct { + name string + entries []*logproto.Entry + direction logproto.Direction + from int64 + through int64 + pipeline log.StreamPipeline + expected []*logproto.Entry + }{ + { + name: "Forward direction, all entries within range", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "line 1"}, + {Timestamp: time.Unix(0, 2), Line: "line 2"}, + {Timestamp: time.Unix(0, 3), Line: "line 3"}, + }, + direction: logproto.FORWARD, + from: 0, + through: 4, + pipeline: noopStreamPipeline(), + expected: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "line 1"}, + {Timestamp: time.Unix(0, 2), Line: "line 2"}, + {Timestamp: time.Unix(0, 3), Line: "line 3"}, + }, + }, + { + name: "Backward direction, all entries within range", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "line 1"}, + {Timestamp: time.Unix(0, 2), Line: "line 2"}, + {Timestamp: time.Unix(0, 3), Line: "line 3"}, + }, + direction: logproto.BACKWARD, + from: 0, + through: 4, + pipeline: noopStreamPipeline(), + expected: []*logproto.Entry{ + {Timestamp: time.Unix(0, 3), Line: "line 3"}, + {Timestamp: time.Unix(0, 2), Line: "line 2"}, + {Timestamp: time.Unix(0, 1), Line: "line 1"}, + }, + }, + { + name: "Forward direction, partial range", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "line 1"}, + {Timestamp: time.Unix(0, 2), Line: "line 2"}, + {Timestamp: time.Unix(0, 3), Line: "line 3"}, + {Timestamp: time.Unix(0, 4), Line: "line 4"}, + }, + direction: logproto.FORWARD, + from: 2, + through: 4, + pipeline: noopStreamPipeline(), + expected: []*logproto.Entry{ + {Timestamp: time.Unix(0, 2), Line: "line 2"}, + {Timestamp: time.Unix(0, 3), Line: "line 3"}, + }, + }, + { + name: "Forward direction with logql pipeline filter", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1).UTC(), Line: "error: something went wrong"}, + {Timestamp: time.Unix(0, 2).UTC(), Line: "info: operation successful"}, + {Timestamp: time.Unix(0, 3).UTC(), Line: "error: another error occurred"}, + {Timestamp: time.Unix(0, 4).UTC(), Line: "debug: checking status"}, + }, + direction: logproto.FORWARD, + from: 1, + through: 5, + pipeline: mustNewPipeline(t, `{foo="bar"} | line_format "foo {{ __line__ }}" |= "error"`), + expected: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "foo error: something went wrong"}, + {Timestamp: time.Unix(0, 3), Line: "foo error: another error occurred"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + + // Write the chunk + _, err := WriteChunk(&buf, tt.entries, EncodingSnappy) + require.NoError(t, err, "WriteChunk failed") + + // Create the iterator + iter, err := NewEntryIterator(buf.Bytes(), tt.pipeline, tt.direction, tt.from, tt.through) + require.NoError(t, err, "NewEntryIterator failed") + defer iter.Close() + + // Read entries using the iterator + var actualEntries []*logproto.Entry + for iter.Next() { + entry := iter.At() + actualEntries = append(actualEntries, &logproto.Entry{ + Timestamp: entry.Timestamp, + Line: entry.Line, + }) + } + require.NoError(t, iter.Err(), "Iterator encountered an error") + + // Compare actual entries with expected entries + require.Equal(t, tt.expected, actualEntries, "Entries do not match expected values") + }) + } +} + +// mustNewPipeline creates a new pipeline or fails the test +func mustNewPipeline(t *testing.T, query string) log.StreamPipeline { + t.Helper() + if query == "" { + return log.NewNoopPipeline().ForStream(labels.Labels{}) + } + expr, err := syntax.ParseLogSelector(query, true) + require.NoError(t, err) + + pipeline, err := expr.Pipeline() + require.NoError(t, err) + + return pipeline.ForStream(labels.Labels{}) +} + +func noopStreamPipeline() log.StreamPipeline { + return log.NewNoopPipeline().ForStream(labels.Labels{}) +} diff --git a/pkg/storage/wal/chunks/sample_iterator.go b/pkg/storage/wal/chunks/sample_iterator.go new file mode 100644 index 0000000000000..4d4b397b1dd58 --- /dev/null +++ b/pkg/storage/wal/chunks/sample_iterator.go @@ -0,0 +1,87 @@ +package chunks + +import ( + "github.com/grafana/loki/v3/pkg/iter" + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql/log" +) + +// NewSampleIterator creates an iterator for efficiently traversing samples in a chunk. +// It takes compressed chunk data, a processing pipeline, iteration direction, and a time range. +// The returned iterator filters samples based on the time range and applies the given pipeline. +// It handles both forward and backward iteration. +// +// Parameters: +// - chunkData: Compressed chunk data containing samples +// - pipeline: StreamSampleExtractor for processing and filtering samples +// - from: Start timestamp (inclusive) for filtering samples +// - through: End timestamp (exclusive) for filtering samples +// +// Returns a SampleIterator and an error if creation fails. +func NewSampleIterator( + chunkData []byte, + pipeline log.StreamSampleExtractor, + from, through int64, +) (iter.SampleIterator, error) { + chkReader, err := NewChunkReader(chunkData) + if err != nil { + return nil, err + } + it := &sampleBufferedIterator{ + reader: chkReader, + pipeline: pipeline, + from: from, + through: through, + } + return it, nil +} + +// At implements iter.SampleIterator. +func (s *sampleBufferedIterator) At() logproto.Sample { + return s.cur +} + +// Close implements iter.SampleIterator. +func (s *sampleBufferedIterator) Close() error { + return s.reader.Close() +} + +// Err implements iter.SampleIterator. +func (s *sampleBufferedIterator) Err() error { + return s.reader.Err() +} + +// Labels implements iter.SampleIterator. +func (s *sampleBufferedIterator) Labels() string { + return s.currLabels.String() +} + +// Next implements iter.SampleIterator. +func (s *sampleBufferedIterator) Next() bool { + for s.reader.Next() { + // todo: Only use length columns for bytes_over_time without filter. + ts, line := s.reader.At() + // check if the timestamp is within the range before applying the pipeline. + if ts < s.from { + continue + } + if ts >= s.through { + return false + } + // todo: structured metadata. + val, lbs, matches := s.pipeline.Process(ts, line) + if !matches { + continue + } + s.currLabels = lbs + s.cur.Value = val + s.cur.Timestamp = ts + return true + } + return false +} + +// StreamHash implements iter.SampleIterator. +func (s *sampleBufferedIterator) StreamHash() uint64 { + return s.pipeline.BaseLabels().Hash() +} diff --git a/pkg/storage/wal/chunks/sample_iterator_test.go b/pkg/storage/wal/chunks/sample_iterator_test.go new file mode 100644 index 0000000000000..4e208301ab9d6 --- /dev/null +++ b/pkg/storage/wal/chunks/sample_iterator_test.go @@ -0,0 +1,202 @@ +package chunks + +import ( + "bytes" + "testing" + "time" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/v3/pkg/logproto" + "github.com/grafana/loki/v3/pkg/logql/log" + "github.com/grafana/loki/v3/pkg/logql/syntax" +) + +func TestNewSampleIterator(t *testing.T) { + tests := []struct { + name string + entries []*logproto.Entry + from int64 + through int64 + extractor log.StreamSampleExtractor + expected []logproto.Sample + expectErr bool + }{ + { + name: "All samples within range", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "1.0"}, + {Timestamp: time.Unix(0, 2), Line: "2.0"}, + {Timestamp: time.Unix(0, 3), Line: "3.0"}, + }, + from: 0, + through: 4, + extractor: mustNewExtractor(t, ""), + expected: []logproto.Sample{ + {Timestamp: 1, Value: 1.0, Hash: 0}, + {Timestamp: 2, Value: 1.0, Hash: 0}, + {Timestamp: 3, Value: 1.0, Hash: 0}, + }, + }, + { + name: "Partial range", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "1.0"}, + {Timestamp: time.Unix(0, 2), Line: "2.0"}, + {Timestamp: time.Unix(0, 3), Line: "3.0"}, + {Timestamp: time.Unix(0, 4), Line: "4.0"}, + }, + from: 2, + through: 4, + extractor: mustNewExtractor(t, ""), + expected: []logproto.Sample{ + {Timestamp: 2, Value: 1.0, Hash: 0}, + {Timestamp: 3, Value: 1.0, Hash: 0}, + }, + }, + { + name: "Pipeline filter", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "error: 1.0"}, + {Timestamp: time.Unix(0, 2), Line: "info: 2.0"}, + {Timestamp: time.Unix(0, 3), Line: "error: 3.0"}, + {Timestamp: time.Unix(0, 4), Line: "debug: 4.0"}, + }, + from: 1, + through: 5, + extractor: mustNewExtractor(t, `count_over_time({foo="bar"} |= "error"[1m])`), + expected: []logproto.Sample{ + {Timestamp: 1, Value: 1.0, Hash: 0}, + {Timestamp: 3, Value: 1.0, Hash: 0}, + }, + }, + { + name: "Pipeline filter with bytes_over_time", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "error: 1.0"}, + {Timestamp: time.Unix(0, 2), Line: "info: 2.0"}, + {Timestamp: time.Unix(0, 3), Line: "error: 3.0"}, + {Timestamp: time.Unix(0, 4), Line: "debug: 4.0"}, + }, + from: 1, + through: 5, + extractor: mustNewExtractor(t, `bytes_over_time({foo="bar"} |= "error"[1m])`), + expected: []logproto.Sample{ + {Timestamp: 1, Value: 10, Hash: 0}, + {Timestamp: 3, Value: 10, Hash: 0}, + }, + }, + { + name: "No samples within range", + entries: []*logproto.Entry{ + {Timestamp: time.Unix(0, 1), Line: "1.0"}, + {Timestamp: time.Unix(0, 2), Line: "2.0"}, + }, + from: 3, + through: 5, + extractor: mustNewExtractor(t, ""), + expected: nil, + }, + { + name: "Empty chunk", + entries: []*logproto.Entry{}, + from: 0, + through: 5, + extractor: mustNewExtractor(t, ""), + expected: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + + // Write the chunk + _, err := WriteChunk(&buf, tt.entries, EncodingSnappy) + require.NoError(t, err, "WriteChunk failed") + + // Create the iterator + iter, err := NewSampleIterator(buf.Bytes(), tt.extractor, tt.from, tt.through) + if tt.expectErr { + require.Error(t, err, "Expected an error but got none") + return + } + require.NoError(t, err, "NewSampleIterator failed") + defer iter.Close() + + // Read samples using the iterator + var actualSamples []logproto.Sample + for iter.Next() { + actualSamples = append(actualSamples, iter.At()) + } + require.NoError(t, iter.Err(), "Iterator encountered an error") + + // Compare actual samples with expected samples + require.Equal(t, tt.expected, actualSamples, "Samples do not match expected values") + + // Check labels + if len(actualSamples) > 0 { + require.Equal(t, tt.extractor.BaseLabels().String(), iter.Labels(), "Unexpected labels") + } + + // Check StreamHash + if len(actualSamples) > 0 { + require.Equal(t, tt.extractor.BaseLabels().Hash(), iter.StreamHash(), "Unexpected StreamHash") + } + }) + } +} + +func TestNewSampleIteratorErrors(t *testing.T) { + tests := []struct { + name string + chunkData []byte + extractor log.StreamSampleExtractor + from int64 + through int64 + }{ + { + name: "Invalid chunk data", + chunkData: []byte("invalid chunk data"), + extractor: mustNewExtractor(t, ""), + from: 0, + through: 10, + }, + { + name: "Nil extractor", + chunkData: []byte{}, // valid empty chunk + extractor: nil, + from: 0, + through: 10, + }, + { + name: "Invalid time range", + chunkData: []byte{}, // valid empty chunk + extractor: mustNewExtractor(t, ""), + from: 10, + through: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewSampleIterator(tt.chunkData, tt.extractor, tt.from, tt.through) + require.Error(t, err, "Expected an error but got none") + }) + } +} + +func mustNewExtractor(t *testing.T, query string) log.StreamSampleExtractor { + t.Helper() + if query == `` { + query = `count_over_time({foo="bar"}[1m])` + } + expr, err := syntax.ParseSampleExpr(query) + require.NoError(t, err) + + extractor, err := expr.Extractor() + require.NoError(t, err) + + return extractor.ForStream(labels.Labels{}) +} diff --git a/pkg/storage/wal/index/index.go b/pkg/storage/wal/index/index.go index 29436bd2044b8..8959824c92780 100644 --- a/pkg/storage/wal/index/index.go +++ b/pkg/storage/wal/index/index.go @@ -24,6 +24,8 @@ import ( "math" "slices" "sort" + "strings" + "unicode/utf8" "unsafe" "github.com/prometheus/prometheus/model/labels" @@ -51,8 +53,24 @@ const ( // checkContextEveryNIterations is used in some tight loops to check if the context is done. checkContextEveryNIterations = 128 + + TenantLabel = "__loki_tenant__" ) +// Bitmap used by func isRegexMetaCharacter to check whether a character needs to be escaped. +var regexMetaCharacterBytes [16]byte + +// isRegexMetaCharacter reports whether byte b needs to be escaped. +func isRegexMetaCharacter(b byte) bool { + return b < utf8.RuneSelf && regexMetaCharacterBytes[b%16]&(1<<(b/16)) != 0 +} + +func init() { + for _, b := range []byte(`.+*?()|[]{}^$`) { + regexMetaCharacterBytes[b%16] |= 1 << (b / 16) + } +} + var AllPostingsKey = labels.Label{} type indexWriterSeries struct { @@ -1908,3 +1926,256 @@ func (dec *Decoder) Series(b []byte, builder *labels.ScratchBuilder, chks *[]chu func yoloString(b []byte) string { return *((*string)(unsafe.Pointer(&b))) } + +// PostingsForMatchers assembles a single postings iterator against the index reader +// based on the given matchers. The resulting postings are not ordered by series. +func (r *Reader) PostingsForMatchers(ctx context.Context, ms ...*labels.Matcher) (index.Postings, error) { + var its, notIts []index.Postings + // See which label must be non-empty. + // Optimization for case like {l=~".", l!="1"}. + labelMustBeSet := make(map[string]bool, len(ms)) + for _, m := range ms { + if !m.Matches("") { + labelMustBeSet[m.Name] = true + } + } + isSubtractingMatcher := func(m *labels.Matcher) bool { + if !labelMustBeSet[m.Name] { + return true + } + return (m.Type == labels.MatchNotEqual || m.Type == labels.MatchNotRegexp) && m.Matches("") + } + hasSubtractingMatchers, hasIntersectingMatchers := false, false + for _, m := range ms { + if isSubtractingMatcher(m) { + hasSubtractingMatchers = true + } else { + hasIntersectingMatchers = true + } + } + + if hasSubtractingMatchers && !hasIntersectingMatchers { + // If there's nothing to subtract from, add in everything and remove the notIts later. + // We prefer to get AllPostings so that the base of subtraction (i.e. allPostings) + // doesn't include series that may be added to the index reader during this function call. + k, v := index.AllPostingsKey() + allPostings, err := r.Postings(ctx, k, v) + if err != nil { + return nil, err + } + its = append(its, allPostings) + } + + // Sort matchers to have the intersecting matchers first. + // This way the base for subtraction is smaller and + // there is no chance that the set we subtract from + // contains postings of series that didn't exist when + // we constructed the set we subtract by. + slices.SortStableFunc(ms, func(i, j *labels.Matcher) int { + if !isSubtractingMatcher(i) && isSubtractingMatcher(j) { + return -1 + } + + return +1 + }) + + for _, m := range ms { + if ctx.Err() != nil { + return nil, ctx.Err() + } + switch { + case m.Name == "" && m.Value == "": // Special-case for AllPostings, used in tests at least. + k, v := index.AllPostingsKey() + allPostings, err := r.Postings(ctx, k, v) + if err != nil { + return nil, err + } + its = append(its, allPostings) + case labelMustBeSet[m.Name]: + // If this matcher must be non-empty, we can be smarter. + matchesEmpty := m.Matches("") + isNot := m.Type == labels.MatchNotEqual || m.Type == labels.MatchNotRegexp + switch { + case isNot && matchesEmpty: // l!="foo" + // If the label can't be empty and is a Not and the inner matcher + // doesn't match empty, then subtract it out at the end. + inverse, err := m.Inverse() + if err != nil { + return nil, err + } + + it, err := postingsForMatcher(ctx, r, inverse) + if err != nil { + return nil, err + } + notIts = append(notIts, it) + case isNot && !matchesEmpty: // l!="" + // If the label can't be empty and is a Not, but the inner matcher can + // be empty we need to use inversePostingsForMatcher. + inverse, err := m.Inverse() + if err != nil { + return nil, err + } + + it, err := inversePostingsForMatcher(ctx, r, inverse) + if err != nil { + return nil, err + } + if index.IsEmptyPostingsType(it) { + return index.EmptyPostings(), nil + } + its = append(its, it) + default: // l="a" + // Non-Not matcher, use normal postingsForMatcher. + it, err := postingsForMatcher(ctx, r, m) + if err != nil { + return nil, err + } + if index.IsEmptyPostingsType(it) { + return index.EmptyPostings(), nil + } + its = append(its, it) + } + default: // l="" + // If the matchers for a labelname selects an empty value, it selects all + // the series which don't have the label name set too. See: + // https://github.com/prometheus/prometheus/issues/3575 and + // https://github.com/prometheus/prometheus/pull/3578#issuecomment-351653555 + it, err := inversePostingsForMatcher(ctx, r, m) + if err != nil { + return nil, err + } + notIts = append(notIts, it) + } + } + + it := index.Intersect(its...) + + for _, n := range notIts { + it = index.Without(it, n) + } + + return it, nil +} + +// inversePostingsForMatcher returns the postings for the series with the label name set but not matching the matcher. +func inversePostingsForMatcher(ctx context.Context, ix *Reader, m *labels.Matcher) (index.Postings, error) { + // Fast-path for MatchNotRegexp matching. + // Inverse of a MatchNotRegexp is MatchRegexp (double negation). + // Fast-path for set matching. + if m.Type == labels.MatchNotRegexp { + setMatches := findSetMatches(m.GetRegexString()) + if len(setMatches) > 0 { + return ix.Postings(ctx, m.Name, setMatches...) + } + } + + // Fast-path for MatchNotEqual matching. + // Inverse of a MatchNotEqual is MatchEqual (double negation). + if m.Type == labels.MatchNotEqual { + return ix.Postings(ctx, m.Name, m.Value) + } + + vals, err := ix.LabelValues(ctx, m.Name) + if err != nil { + return nil, err + } + + var res []string + // If the inverse match is ="", we just want all the values. + if m.Type == labels.MatchEqual && m.Value == "" { + res = vals + } else { + for _, val := range vals { + if !m.Matches(val) { + res = append(res, val) + } + } + } + + return ix.Postings(ctx, m.Name, res...) +} + +func postingsForMatcher(ctx context.Context, ix *Reader, m *labels.Matcher) (index.Postings, error) { + // This method will not return postings for missing labels. + + // Fast-path for equal matching. + if m.Type == labels.MatchEqual { + return ix.Postings(ctx, m.Name, m.Value) + } + + // Fast-path for set matching. + if m.Type == labels.MatchRegexp { + setMatches := findSetMatches(m.GetRegexString()) + if len(setMatches) > 0 { + return ix.Postings(ctx, m.Name, setMatches...) + } + } + + vals, err := ix.LabelValues(ctx, m.Name) + if err != nil { + return nil, err + } + + var res []string + for _, val := range vals { + if m.Matches(val) { + res = append(res, val) + } + } + + if len(res) == 0 { + return index.EmptyPostings(), nil + } + + return ix.Postings(ctx, m.Name, res...) +} + +func findSetMatches(pattern string) []string { + // Return empty matches if the wrapper from Prometheus is missing. + if len(pattern) < 6 || pattern[:4] != "^(?:" || pattern[len(pattern)-2:] != ")$" { + return nil + } + escaped := false + sets := []*strings.Builder{{}} + init := 4 + end := len(pattern) - 2 + // If the regex is wrapped in a group we can remove the first and last parentheses + if pattern[init] == '(' && pattern[end-1] == ')' { + init++ + end-- + } + for i := init; i < end; i++ { + if escaped { + switch { + case isRegexMetaCharacter(pattern[i]): + sets[len(sets)-1].WriteByte(pattern[i]) + case pattern[i] == '\\': + sets[len(sets)-1].WriteByte('\\') + default: + return nil + } + escaped = false + } else { + switch { + case isRegexMetaCharacter(pattern[i]): + if pattern[i] == '|' { + sets = append(sets, &strings.Builder{}) + } else { + return nil + } + case pattern[i] == '\\': + escaped = true + default: + sets[len(sets)-1].WriteByte(pattern[i]) + } + } + } + matches := make([]string, 0, len(sets)) + for _, s := range sets { + if s.Len() > 0 { + matches = append(matches, s.String()) + } + } + return matches +} diff --git a/pkg/storage/wal/manager.go b/pkg/storage/wal/manager.go index fc23cb21e742f..1a7b73047bbdb 100644 --- a/pkg/storage/wal/manager.go +++ b/pkg/storage/wal/manager.go @@ -156,7 +156,7 @@ func (m *Manager) Append(r AppendRequest) (*AppendResult, error) { s.w.Append(r.TenantID, r.LabelsStr, r.Labels, r.Entries, m.clock.Now()) // If the segment exceeded the maximum age or the maximum size, move s to // the closed list to be flushed. - if m.clock.Since(s.w.firstAppend) >= m.cfg.MaxAge || s.w.InputSize() >= m.cfg.MaxSegmentSize { + if s.w.Age(m.clock.Now()) >= m.cfg.MaxAge || s.w.InputSize() >= m.cfg.MaxSegmentSize { m.move(el, s) } return s.r, nil @@ -224,7 +224,7 @@ func (m *Manager) move(el *list.Element, s *segment) { func (m *Manager) moveFrontIfExpired() bool { if el := m.available.Front(); el != nil { s := el.Value.(*segment) - if !s.w.firstAppend.IsZero() && m.clock.Since(s.w.firstAppend) >= m.cfg.MaxAge { + if s.w.Age(m.clock.Now()) >= m.cfg.MaxAge { m.move(el, s) return true } diff --git a/pkg/storage/wal/manager_test.go b/pkg/storage/wal/manager_test.go index 93e10fbaa06a9..1a14d999f5ecd 100644 --- a/pkg/storage/wal/manager_test.go +++ b/pkg/storage/wal/manager_test.go @@ -54,6 +54,38 @@ func TestManager_Append(t *testing.T) { require.NoError(t, res.Err()) } +func TestManager_AppendNoEntries(t *testing.T) { + m, err := NewManager(Config{ + MaxAge: 30 * time.Second, + MaxSegments: 1, + MaxSegmentSize: 1024, // 1KB + }, NewManagerMetrics(nil)) + require.NoError(t, err) + + // Append no entries. + lbs := labels.Labels{{Name: "a", Value: "b"}} + res, err := m.Append(AppendRequest{ + TenantID: "1", + Labels: lbs, + LabelsStr: lbs.String(), + Entries: []*logproto.Entry{}, + }) + require.NoError(t, err) + require.NotNil(t, res) + + // The data hasn't been flushed, so reading from Done() should block. + select { + case <-res.Done(): + t.Fatal("unexpected closed Done()") + default: + } + + // The segment that was just appended to has neither reached the maximum + // age nor maximum size to be flushed. + require.Equal(t, 1, m.available.Len()) + require.Equal(t, 0, m.pending.Len()) +} + func TestManager_AppendFailed(t *testing.T) { m, err := NewManager(Config{ MaxAge: 30 * time.Second, diff --git a/pkg/storage/wal/segment.go b/pkg/storage/wal/segment.go index 5922d38fe7395..93b824bbcb70e 100644 --- a/pkg/storage/wal/segment.go +++ b/pkg/storage/wal/segment.go @@ -35,7 +35,7 @@ var ( } }, } - tenantLabel = "__loki_tenant__" + Dir = "loki-v2/wal/anon/" ) func init() { @@ -140,6 +140,9 @@ func NewWalSegmentWriter() (*SegmentWriter, error) { // Age returns the age of the segment. func (b *SegmentWriter) Age(now time.Time) time.Duration { + if b.firstAppend.IsZero() { + return 0 + } return now.Sub(b.firstAppend) } @@ -153,8 +156,8 @@ func (b *SegmentWriter) getOrCreateStream(id streamID, lbls labels.Labels) *stre if ok { return s } - if lbls.Get(tenantLabel) == "" { - lbls = labels.NewBuilder(lbls).Set(tenantLabel, id.tenant).Labels() + if lbls.Get(index.TenantLabel) == "" { + lbls = labels.NewBuilder(lbls).Set(index.TenantLabel, id.tenant).Labels() } s = streamSegmentPool.Get().(*streamSegment) s.lbls = lbls diff --git a/pkg/storage/wal/segment_test.go b/pkg/storage/wal/segment_test.go index 90755adcfcc3f..cbe42587bf7a2 100644 --- a/pkg/storage/wal/segment_test.go +++ b/pkg/storage/wal/segment_test.go @@ -122,7 +122,7 @@ func TestWalSegmentWriter_Append(t *testing.T) { require.True(t, ok) lbs, err := syntax.ParseLabels(expected.labels) require.NoError(t, err) - lbs = append(lbs, labels.Label{Name: tenantLabel, Value: expected.tenant}) + lbs = append(lbs, labels.Label{Name: index.TenantLabel, Value: expected.tenant}) sort.Sort(lbs) require.Equal(t, lbs, stream.lbls) require.Equal(t, expected.entries, stream.entries) @@ -168,7 +168,7 @@ func TestMultiTenantWrite(t *testing.T) { for _, tenant := range tenants { for _, lbl := range lbls { - expectedSeries = append(expectedSeries, labels.NewBuilder(lbl).Set(tenantLabel, tenant).Labels().String()) + expectedSeries = append(expectedSeries, labels.NewBuilder(lbl).Set(index.TenantLabel, tenant).Labels().String()) } } diff --git a/pkg/util/httpreq/tags.go b/pkg/util/httpreq/tags.go index a0ed6c0d05386..96e4091c7db42 100644 --- a/pkg/util/httpreq/tags.go +++ b/pkg/util/httpreq/tags.go @@ -40,6 +40,12 @@ func ExtractQueryTagsFromHTTP(req *http.Request) string { return safeQueryTags.ReplaceAllString(tags, "_") } +func ExtractQueryTagsFromContext(ctx context.Context) string { + // if the cast fails then v will be an empty string + v, _ := ctx.Value(QueryTagsHTTPHeader).(string) + return v +} + func InjectQueryTags(ctx context.Context, tags string) context.Context { tags = safeQueryTags.ReplaceAllString(tags, "_") return context.WithValue(ctx, QueryTagsHTTPHeader, tags) diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index 6e3b8f17a9d63..b07cf4ceafd1c 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -13,6 +13,14 @@ Entries should include a reference to the pull request that introduced the chang [//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +## 6.8.0 + +- [BUGFIX] Fixed how we set imagePullSecrets for the admin-api and enterprise-gateway + +## 6.7.4 + +- [ENHANCEMENT] Allow configuring the SSE section under AWS S3 storage config. + ## 6.7.3 - [BUGFIX] Removed Helm test binary diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 641a28a425edb..c8bc875044569 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. type: application appVersion: 3.1.0 -version: 6.7.3 +version: 6.8.0 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index 24f84ace97212..7f3f08bafd49a 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 6.7.3](https://img.shields.io/badge/Version-6.7.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.1.0](https://img.shields.io/badge/AppVersion-3.1.0-informational?style=flat-square) +![Version: 6.8.0](https://img.shields.io/badge/Version-6.8.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.1.0](https://img.shields.io/badge/AppVersion-3.1.0-informational?style=flat-square) Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index 8d4a0a9cb94ef..91b453efa062a 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -239,30 +239,15 @@ s3: insecure: {{ .insecure }} {{- with .http_config}} http_config: - {{- with .idle_conn_timeout }} - idle_conn_timeout: {{ . }} - {{- end}} - {{- with .response_header_timeout }} - response_header_timeout: {{ . }} - {{- end}} - {{- with .insecure_skip_verify }} - insecure_skip_verify: {{ . }} - {{- end}} - {{- with .ca_file}} - ca_file: {{ . }} - {{- end}} +{{ toYaml . | indent 4 }} {{- end }} {{- with .backoff_config}} backoff_config: - {{- with .min_period }} - min_period: {{ . }} - {{- end}} - {{- with .max_period }} - max_period: {{ . }} - {{- end}} - {{- with .max_retries }} - max_retries: {{ . }} - {{- end}} +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .sse }} + sse: +{{ toYaml . | indent 4 }} {{- end }} {{- end -}} @@ -308,35 +293,7 @@ alibabacloud: {{- else if eq .Values.loki.storage.type "swift" -}} {{- with .Values.loki.storage.swift }} swift: - {{- with .auth_version }} - auth_version: {{ . }} - {{- end }} - auth_url: {{ .auth_url }} - {{- with .internal }} - internal: {{ . }} - {{- end }} - username: {{ .username }} - user_domain_name: {{ .user_domain_name }} - {{- with .user_domain_id }} - user_domain_id: {{ . }} - {{- end }} - {{- with .user_id }} - user_id: {{ . }} - {{- end }} - password: {{ .password }} - {{- with .domain_id }} - domain_id: {{ . }} - {{- end }} - domain_name: {{ .domain_name }} - project_id: {{ .project_id }} - project_name: {{ .project_name }} - project_domain_id: {{ .project_domain_id }} - project_domain_name: {{ .project_domain_name }} - region_name: {{ .region_name }} - container_name: {{ .container_name }} - max_retries: {{ .max_retries | default 3 }} - connect_timeout: {{ .connect_timeout | default "10s" }} - request_timeout: {{ .request_timeout | default "5s" }} +{{ toYaml . | indent 2 }} {{- end -}} {{- else -}} {{- with .Values.loki.storage.filesystem }} diff --git a/production/helm/loki/templates/admin-api/deployment-admin-api.yaml b/production/helm/loki/templates/admin-api/deployment-admin-api.yaml index 15391665ca776..650c72fc15983 100644 --- a/production/helm/loki/templates/admin-api/deployment-admin-api.yaml +++ b/production/helm/loki/templates/admin-api/deployment-admin-api.yaml @@ -65,11 +65,9 @@ spec: mountPath: {{ .Values.minio.configPathmc }}certs {{ end }} {{- end }} - {{- if .Values.imagePullSecrets }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} + {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.adminApi.hostAliases }} hostAliases: diff --git a/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml b/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml index de8ba11058eb1..746fa6142b771 100644 --- a/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml +++ b/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml @@ -46,11 +46,9 @@ spec: {{- toYaml .Values.enterpriseGateway.podSecurityContext | nindent 8 }} initContainers: {{- toYaml .Values.enterpriseGateway.initContainers | nindent 8 }} - {{- if .Values.imagePullSecrets }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} + {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.enterpriseGateway.hostAliases }} hostAliases: