Merge branch 'main' into log-4550-gcp

grafana · Nov 24, 2023 · 1af852e · 1af852e
2 parents e816a3c + d62d4e3
commit 1af852e
Show file tree

Hide file tree

Showing 324 changed files with 12,488 additions and 7,487 deletions.
diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
@@ -160,14 +160,14 @@ local promtail_win() = pipeline('promtail-windows') {
   steps: [
     {
       name: 'identify-runner',
-      image: 'golang:1.19-windowsservercore-1809',
+      image: 'golang:1.21.3-windowsservercore-1809',
       commands: [
         'Write-Output $env:DRONE_RUNNER_NAME',
       ],
     },
     {
       name: 'test',
-      image: 'golang:1.19-windowsservercore-1809',
+      image: 'golang:1.21.3-windowsservercore-1809',
       commands: [
         'go test .\\clients\\pkg\\promtail\\targets\\windows\\... -v',
       ],

diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -1666,11 +1666,11 @@ platform:
 steps:
 - commands:
   - Write-Output $env:DRONE_RUNNER_NAME
-  image: golang:1.19-windowsservercore-1809
+  image: golang:1.21.3-windowsservercore-1809
   name: identify-runner
 - commands:
   - go test .\clients\pkg\promtail\targets\windows\... -v
-  image: golang:1.19-windowsservercore-1809
+  image: golang:1.21.3-windowsservercore-1809
   name: test
 trigger:
   ref:
@@ -2106,6 +2106,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: caf375427f92f78711f801f56341357b67737330e906346ee908a796c61dd314
+hmac: 8ae9cff1a379503d0b568f727d9c12bcb486a5e8d1fc3271deea32f07939baf1
 
 ...
diff --git a/.github/workflows/nix-ci.yaml b/.github/workflows/nix-ci.yaml
@@ -6,12 +6,30 @@ on:
       - "flake.nix"
       - "nix/**"
 jobs:
-  tests:
+  lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v22
         with:
           nix_path: nixpkgs=channel:nixos-unstable
-      - run: nix run .#lint
-      - run: nix build --print-build-logs
+      - run: nix run --print-build-logs .#lint
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v22
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix run --print-build-logs .#test
+  packages:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v22
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix build --print-build-logs .#logcli
+      - run: nix build --print-build-logs .#loki
+      - run: nix build --print-build-logs .#loki-canary
+      - run: nix build --print-build-logs .#promtail
diff --git a/.github/workflows/snyk-pr-comment.yml b/.github/workflows/snyk-pr-comment.yml
@@ -0,0 +1,47 @@
+name: PR Vulnerability Scan
+on: pull_request
+
+permissions:
+  pull-requests: write
+  issues: write
+
+jobs:
+  snyk:
+    name: Snyk Scan
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.head.repo.fork }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/golang@master
+        continue-on-error: true # To make sure that PR comment is made
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          command: test
+          args: --severity-threshold=high --json-file-output=snyk.json
+
+      - name: Prepare Snyk message
+        run: |
+          echo "Snyk scan found the following vulnerabilities:" > snyk.txt
+
+      - name: Format Snyk Message
+        uses: sergeysova/jq-action@v2
+        continue-on-error: true
+        with:
+          cmd: jq -r  '.vulnerabilities[] | "* **\(.severity)** - [\(.identifiers.CVE[0])] \(.title) in `\(.moduleName)` v\(.version). Fixed in \(.fixedIn)"' snyk.json >> snyk.txt
+
+      - name: Determine whether to comment
+        continue-on-error: true
+        id: should-comment
+        run: |
+          if [[ $(wc -l < snyk.txt) -gt 1 ]]; then exit 0; fi
+          exit 1
+
+      - name: Comment on PR with Snyk scan results
+        uses: mshick/add-pr-comment@v2
+        if: ${{ steps.should-comment.outcome == 'success' }}
+        with:
+          message-id: snyk-${{ github.event.number }}
+          message-path: snyk.txt
diff --git a/.github/workflows/vulnerability-scan.yml → .github/workflows/trivy-pr-comment.yml b/.github/workflows/vulnerability-scan.yml → .github/workflows/trivy-pr-comment.yml
@@ -1,58 +1,14 @@
 name: PR Vulnerability Scan
-on: pull_request
+on: pull_request_target
 
 permissions:
   pull-requests: write
-  contents: write
+  issues: write
 
 jobs:
-  snyk:
-    name: Snyk Scan
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@master
-      - name: Run Snyk to check for vulnerabilities
-        uses: snyk/actions/golang@master
-        continue-on-error: true # To make sure that PR comment is made
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          command: test
-          args: --severity-threshold=high --json-file-output=snyk.json
-
-      - name: Prepare Snyk message
-        run: |
-          echo "Snyk scan found the following vulnerabilities:" > snyk.txt
-
-      - name: Format Snyk Message
-        uses: sergeysova/jq-action@v2
-        continue-on-error: true
-        with:
-          cmd: jq -r  '.vulnerabilities[] | "* **\(.severity)** - [\(.identifiers.CVE[0])] \(.title) in `\(.moduleName)` v\(.version). Fixed in \(.fixedIn)"' snyk.json >> snyk.txt
-
-      - name: Determine whether to comment
-        continue-on-error: true
-        id: should-comment
-        run: |
-          if [[ $(wc -l < snyk.txt) -gt 1 ]]; then exit 0; fi
-          exit 1
-
-      - name: Comment on PR with Snyk scan results
-        uses: mshick/add-pr-comment@v2
-        if: ${{ steps.should-comment.outcome == 'success' }}
-        with:
-          message-id: snyk-${{ github.event.number }}
-          message-path: snyk.txt
   trivy:
     name: Trivy Scan
     runs-on: ubuntu-20.04
-    permissions:
-      issues: write
-      pull-requests: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

diff --git a/.golangci.yml b/.golangci.yml
@@ -63,6 +63,10 @@ linters-settings:
           - desc: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
             pkg: github.com/go-kit/kit/log
 
+  misspell:
+    ignore-words:
+      - strat
+
 linters:
   enable:
     - errcheck

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 ##### Enhancements
 
+* [11243](https://github.com/grafana/loki/pull/11243) **kavirajk**: Inflight-logging: Add extra metadata to inflight requests logging.
 * [11110](https://github.com/grafana/loki/pull/11003) **MichelHollands**: Change the default of the `metrics-namespace` flag to 'loki'.
 * [11086](https://github.com/grafana/loki/pull/11086) **kandrew5**: Helm: Allow topologySpreadConstraints
 * [11003](https://github.com/grafana/loki/pull/11003) **MichelHollands**: Add the `metrics-namespace` flag to change the namespace of metrics currently using cortex as namespace.
@@ -39,6 +40,7 @@
 
 ##### Fixes
 * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var.
+* [11195](https://github.com/grafana/loki/pull/11195) **canuteson** Generate tsdb_shipper storage_config even if using_boltdb_shipper is false
 
 ##### Changes
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -149,4 +149,11 @@ To get a local preview of the documentation:
 3. Run the command `make docs`. This uses the `grafana/docs` image which internally uses Hugo to generate the static site.
 4. Open http://localhost:3002/docs/loki/latest/ to review your changes.
 
+**Remember:** If running `make docs` command gave you the following error.
+
+   - `path /tmp/make-docs.Dcq is not shared from the host and is not known to Docker.`
+   - `You can configure shared paths from Docker -> Preferences... -> Resources -> File Sharing.`
+
+Then you can go to Docker Desktop settings and open the resources, add the temporary directory path `/tmp`.
+
 > Note that `make docs` uses a lot of memory. If it crashes, increase the memory allocated to Docker and try again.
diff --git a/Makefile b/Makefile
@@ -42,6 +42,8 @@ BUILD_IMAGE_VERSION ?= 0.31.2
 # Docker image info
 IMAGE_PREFIX ?= grafana
 
+BUILD_IMAGE_PREFIX ?= grafana
+
 IMAGE_TAG ?= $(shell ./tools/image-tag)
 
 # Version info for binaries
@@ -102,7 +104,7 @@ RM := --rm
 TTY := --tty
 
 DOCKER_BUILDKIT ?= 1
-BUILD_IMAGE = BUILD_IMAGE=$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION)
+BUILD_IMAGE = BUILD_IMAGE=$(BUILD_IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION)
 PUSH_OCI=docker push
 TAG_OCI=docker tag
 ifeq ($(CI), true)

diff --git a/clients/cmd/docker-driver/Dockerfile b/clients/cmd/docker-driver/Dockerfile
@@ -9,7 +9,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false clients/cmd/docker-driver/docker-driver
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates tzdata
 COPY --from=build /src/loki/clients/cmd/docker-driver/docker-driver /bin/docker-driver
 WORKDIR /bin/

diff --git a/clients/cmd/promtail/Dockerfile.debug b/clients/cmd/promtail/Dockerfile.debug
@@ -9,7 +9,7 @@ WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false PROMTAIL_JOURNAL_ENABLED=true promtail-debug
 
 
-FROM       alpine:3.18.3
+FROM       alpine:3.18.4
 RUN        apk add --update --no-cache ca-certificates tzdata
 COPY       --from=build /src/loki/clients/cmd/promtail/promtail-debug /usr/bin/promtail-debug
 COPY       --from=build /usr/bin/dlv /usr/bin/dlv

diff --git a/clients/pkg/logentry/logql/parser.go b/clients/pkg/logentry/logql/parser.go
@@ -1,13 +1,14 @@
 package logql
 
 import (
-	"errors"
 	"fmt"
 	"strconv"
 	"strings"
 	"text/scanner"
 
 	"github.com/prometheus/prometheus/model/labels"
+
+	"github.com/grafana/loki/pkg/logqlmodel"
 )
 
 func init() {
@@ -44,7 +45,7 @@ func ParseMatchers(input string) ([]*labels.Matcher, error) {
 	}
 	matcherExpr, ok := expr.(*matchersExpr)
 	if !ok {
-		return nil, errors.New("only label matchers is supported")
+		return nil, logqlmodel.ErrParseMatchers
 	}
 	return matcherExpr.matchers, nil
 }

diff --git a/cmd/logcli/Dockerfile b/cmd/logcli/Dockerfile
@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false logcli
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates
 

diff --git a/cmd/logql-analyzer/Dockerfile b/cmd/logql-analyzer/Dockerfile
@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && CGO_ENABLED=0 go build ./cmd/logql-analyzer/
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates
 

diff --git a/cmd/loki-canary-boringcrypto/Dockerfile b/cmd/loki-canary-boringcrypto/Dockerfile
@@ -5,7 +5,7 @@ WORKDIR /src/loki
 RUN go env GOARCH > /goarch
 RUN make clean && make GOARCH=$(cat /goarch) BUILD_IN_CONTAINER=true GOEXPERIMENT=boringcrypto loki-canary-boringcrypto
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 RUN apk add --no-cache libc6-compat
 COPY --from=build /src/loki/cmd/loki-canary-boringcrypto/loki-canary-boringcrypto /usr/bin/loki-canary

diff --git a/cmd/loki-canary/Dockerfile b/cmd/loki-canary/Dockerfile
@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false loki-canary
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary
 ENTRYPOINT [ "/usr/bin/loki-canary" ]
diff --git a/cmd/loki-canary/Dockerfile.cross b/cmd/loki-canary/Dockerfile.cross
@@ -12,7 +12,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-canary
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary
 ENTRYPOINT [ "/usr/bin/loki-canary" ]
diff --git a/cmd/loki/Dockerfile b/cmd/loki/Dockerfile
@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false loki
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates libcap
 

diff --git a/cmd/loki/Dockerfile.cross b/cmd/loki/Dockerfile.cross
@@ -12,7 +12,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 
 RUN apk add --no-cache ca-certificates
 

diff --git a/cmd/loki/Dockerfile.debug b/cmd/loki/Dockerfile.debug
@@ -15,7 +15,7 @@ WORKDIR /src/loki
 RUN make clean && \
     GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-debug
 
-FROM       alpine:3.18.3
+FROM       alpine:3.18.4
 RUN        apk add --update --no-cache ca-certificates
 COPY       --from=build /src/loki/cmd/loki/loki-debug /usr/bin/loki-debug
 COPY       --from=goenv /go/bin/dlv /usr/bin/dlv

diff --git a/cmd/migrate/Dockerfile b/cmd/migrate/Dockerfile
@@ -3,7 +3,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false migrate
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/migrate/migrate /usr/bin/migrate
 #ENTRYPOINT [ "/usr/bin/migrate" ]

diff --git a/cmd/querytee/Dockerfile b/cmd/querytee/Dockerfile
@@ -4,7 +4,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && make BUILD_IN_CONTAINER=false loki-querytee
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/querytee/querytee /usr/bin/querytee
 ENTRYPOINT [ "/usr/bin/querytee" ]
diff --git a/cmd/querytee/Dockerfile.cross b/cmd/querytee/Dockerfile.cross
@@ -12,7 +12,7 @@ COPY . /src/loki
 WORKDIR /src/loki
 RUN make clean && GOARCH=$(cat /goarch) GOARM=$(cat /goarm) make BUILD_IN_CONTAINER=false loki-querytee
 
-FROM alpine:3.18.3
+FROM alpine:3.18.4
 RUN apk add --update --no-cache ca-certificates
 COPY --from=build /src/loki/cmd/querytee/querytee /usr/bin/querytee
 ENTRYPOINT [ "/usr/bin/querytee" ]
diff --git a/docs/sources/alert/_index.md b/docs/sources/alert/_index.md
@@ -155,7 +155,7 @@ At the time of writing, these are the compatible backends that support this:
 - [Grafana Mimir](/docs/mimir/latest/operators-guide/reference-http-api/#remote-write)
 - [Thanos (`Receiver`)](https://thanos.io/tip/components/receive.md/)
 
-Here is an example remote-write configuration for sending to a local Prometheus instance:
+Here is an example of a remote-write configuration for sending data to a local Prometheus instance:
 
 ```yaml
 ruler: