Merge branch 'main' into paul1r/bloom_updates

.github/workflows/vulnerability-scan.yml

@@ -0,0 +1,90 @@
+name: PR Vulnerability Scan
+on: pull_request
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  snyk:
+    name: Snyk Scan
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/golang@master
+        continue-on-error: true # To make sure that PR comment is made
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          command: test
+          args: --severity-threshold=high --json-file-output=snyk.json
+
+      - name: Prepare Snyk message
+        run: |
+          echo "Snyk scan found the following vulnerabilities:" > snyk.txt
+
+      - name: Format Snyk Message
+        uses: sergeysova/jq-action@v2
+        continue-on-error: true
+        with:
+          cmd: jq -r  '.vulnerabilities[] | "* **\(.severity)** - [\(.identifiers.CVE[0])] \(.title) in `\(.moduleName)` v\(.version). Fixed in \(.fixedIn)"' snyk.json >> snyk.txt
+
+      - name: Determine whether to comment
+        continue-on-error: true
+        id: should-comment
+        run: |
+          if [[ $(wc -l < snyk.txt) -gt 1 ]]; then exit 0; fi
+          exit 1
+
+      - name: Comment on PR with Snyk scan results
+        uses: mshick/add-pr-comment@v2
+        if: ${{ steps.should-comment.outcome == 'success' }}
+        with:
+          message-id: snyk-${{ github.event.number }}
+          message-path: snyk.txt
+  trivy:
+    name: Trivy Scan
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Build Loki Image
+        run: |
+          IMAGE_TAG="$(./tools/image-tag)"
+          make loki-image
+          echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "docker.io/grafana/loki:${{ env.IMAGE_TAG }}"
+          format: "json"
+          output: "trivy.json"
+          severity: "CRITICAL,HIGH"
+
+      - name: Prepare Trivy Message
+        run: |
+          echo "Trivy scan found the following vulnerabilities:" > trivy.txt
+
+      - name: Format Trivy Message
+        uses: sergeysova/jq-action@v2
+        continue-on-error: true
+        with:
+          cmd: jq -r '.Results[] | select(.Vulnerabilities != null) | .Vulnerabilities[] | "* **\(.Severity)** [\(.Title)](\(.PrimaryURL)) in `\(.PkgName)` v\(.InstalledVersion). Fixed in v\(.FixedVersion)"' trivy.json >> trivy.txt
+
+      - name: Determine whether to comment
+        continue-on-error: true
+        id: should-comment
+        run: |
+          if [[ $(wc -l < trivy.txt) -gt 1 ]]; then exit 0; fi
+          exit 1
+
+      - name: Comment on PR with Trivy scan results
+        uses: mshick/add-pr-comment@v2
+        if: ${{ steps.should-comment.outcome == 'success' }}
+        with:
+          message-id: trivy-${{ github.event.number }}
+          message-path: trivy.txt

docs/sources/setup/install/helm/reference.md

@@ -1046,6 +1046,15 @@ true
 			<td><pre lang="json">
 []
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>gateway.extraContainers</td>
+			<td>list</td>
+			<td>Containers to add to the gateway pods</td>
+			<td><pre lang="json">
+[]
+</pre>
 </td>
 		</tr>
 		<tr>
@@ -1260,6 +1269,15 @@ See values.yaml
 			<td><pre lang="json">
 "main '$remote_addr - $remote_user [$time_local]  $status '\n        '\"$request\" $body_bytes_sent \"$http_referer\" '\n        '\"$http_user_agent\" \"$http_x_forwarded_for\"';"
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>gateway.nginxConfig.resolver</td>
+			<td>string</td>
+			<td>Allows overriding the DNS resolver address nginx will use.</td>
+			<td><pre lang="json">
+""
+</pre>
 </td>
 		</tr>
 		<tr>

pkg/querier/queryrange/volume_test.go

@@ -1,15 +1,19 @@
 package queryrange
 
 import (
+	"context"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/grafana/dskit/user"
+
 	"github.com/grafana/loki/pkg/loghttp"
 	"github.com/grafana/loki/pkg/logproto"
 	"github.com/grafana/loki/pkg/push"
 	"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
+	"github.com/grafana/loki/pkg/storage/stores/index/seriesvolume"
 )
 
 const forRangeQuery = false
@@ -250,3 +254,78 @@ func Test_toPrometheusResponse(t *testing.T) {
 		}, promResp.Response.Data)
 	})
 }
+
+func Test_VolumeMiddleware(t *testing.T) {
+	makeVolumeRequest := func(req *logproto.VolumeRequest) *queryrangebase.PrometheusResponse {
+		nextHandler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) {
+			return &VolumeResponse{
+				Response: &logproto.VolumeResponse{
+					Volumes: []logproto.Volume{
+						{
+							Name:   `{foo="bar"}`,
+							Volume: 42,
+						},
+					},
+				},
+			}, nil
+		})
+
+		m := NewVolumeMiddleware()
+		wrapped := m.Wrap(nextHandler)
+
+		ctx := user.InjectOrgID(context.Background(), "fake")
+		resp, err := wrapped.Do(ctx, req)
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		return resp.(*LokiPromResponse).Response
+	}
+
+	t.Run("it breaks query up into subqueries according to step", func(t *testing.T) {
+		volumeReq := &logproto.VolumeRequest{
+			From:        10,
+			Through:     20,
+			Matchers:    `{foo="bar"}`,
+			Limit:       seriesvolume.DefaultLimit,
+			Step:        1,
+			AggregateBy: seriesvolume.Series,
+		}
+		promResp := makeVolumeRequest(volumeReq)
+
+		require.Equal(t, promResp.Data.ResultType, loghttp.ResultTypeMatrix)
+		require.Equal(t, len(promResp.Data.Result), 1)
+		require.Equal(t, len(promResp.Data.Result[0].Samples), 10)
+	})
+
+	t.Run("only returns one datapoint when step is > than time range", func(t *testing.T) {
+		volumeReq := &logproto.VolumeRequest{
+			From:        10,
+			Through:     20,
+			Matchers:    `{foo="bar"}`,
+			Limit:       seriesvolume.DefaultLimit,
+			Step:        20,
+			AggregateBy: seriesvolume.Series,
+		}
+		promResp := makeVolumeRequest(volumeReq)
+
+		require.Equal(t, promResp.Data.ResultType, loghttp.ResultTypeVector)
+		require.Equal(t, len(promResp.Data.Result), 1)
+		require.Equal(t, len(promResp.Data.Result[0].Samples), 1)
+	})
+
+	t.Run("when requested time range is not evenly divisible by step, an extra datpoint is added", func(t *testing.T) {
+		volumeReq := &logproto.VolumeRequest{
+			From:        1698830441000, // 2023-11-01T09:20:41Z
+			Through:     1698830498000, // 2023-11-01T09:21:38Z, difference is 57s
+			Matchers:    `{foo="bar"}`,
+			Limit:       seriesvolume.DefaultLimit,
+			Step:        60000, // 60s
+			AggregateBy: seriesvolume.Series,
+		}
+		promResp := makeVolumeRequest(volumeReq)
+
+		require.Equal(t, promResp.Data.ResultType, loghttp.ResultTypeMatrix)
+		require.Equal(t, 1, len(promResp.Data.Result))
+		require.Equal(t, 2, len(promResp.Data.Result[0].Samples))
+	})
+}

production/helm/loki/CHANGELOG.md

@@ -13,16 +13,18 @@ Entries should include a reference to the pull request that introduced the chang
 
 [//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)
 
+## 5.36.2
+
+- [BUGFIX] Add support to run dnsmasq
+
 ## 5.36.1
 
 - [FEATURE] Allow topology spread constraints for Loki
 
-
 ## 5.36.0
 
 - [CHANGE] Changed version of Loki to 2.9.2
 
-
 ## 5.35.0
 
 - [FEATURE] Add support for configuring distributor.

production/helm/loki/Chart.yaml

@@ -3,7 +3,7 @@ name: loki
 description: Helm chart for Grafana Loki in simple, scalable mode
 type: application
 appVersion: 2.9.2
-version: 5.36.1
+version: 5.36.2
 home: https://grafana.github.io/helm-charts
 sources:
   - https://github.com/grafana/loki

production/helm/loki/README.md

@@ -1,6 +1,6 @@
 # loki
 
-![Version: 5.36.1](https://img.shields.io/badge/Version-5.36.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.9.2](https://img.shields.io/badge/AppVersion-2.9.2-informational?style=flat-square)
+![Version: 5.36.2](https://img.shields.io/badge/Version-5.36.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.9.2](https://img.shields.io/badge/AppVersion-2.9.2-informational?style=flat-square)
 
 Helm chart for Grafana Loki in simple, scalable mode
 

production/helm/loki/templates/_helpers.tpl

@@ -604,7 +604,11 @@ http {
 
   sendfile     on;
   tcp_nopush   on;
+  {{- if .Values.gateway.nginxConfig.resolver }}
+  resolver {{ .Values.gateway.nginxConfig.resolver }};
+  {{- else }}
   resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}.;
+  {{- end }}
 
   {{- with .Values.gateway.nginxConfig.httpSnippet }}
   {{- tpl . $ | nindent 2 }}

production/helm/loki/templates/gateway/deployment-gateway.yaml

@@ -96,6 +96,9 @@ spec:
             {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
+        {{- if .Values.gateway.extraContainers }}
+        {{- toYaml .Values.gateway.extraContainers | nindent 8}}
+        {{- end }}
       {{- with .Values.gateway.affinity }}
       affinity:
         {{- tpl . $ | nindent 8 }}

production/helm/loki/values.yaml

@@ -1324,6 +1324,8 @@ gateway:
     allowPrivilegeEscalation: false
   # -- Resource requests and limits for the gateway
   resources: {}
+  # -- Containers to add to the gateway pods
+  extraContainers: []
   # -- Grace period to allow the gateway to shutdown before it is killed
   terminationGracePeriodSeconds: 30
   # -- Affinity for gateway pods. Passed through `tpl` and, thus, to be configured as string
@@ -1427,6 +1429,8 @@ gateway:
     customWriteUrl: null
     # -- Override Backend URL
     customBackendUrl: null
+    # -- Allows overriding the DNS resolver address nginx will use.
+    resolver: ""
     # -- Config file contents for Nginx. Passed through the `tpl` function to allow templating
     # @default -- See values.yaml
     file: |