Merge branch 'main' into add-mixin-for-backend-resources

.github/workflows/helm-loki-ci.yml

@@ -0,0 +1,211 @@
+---
+name: helm-loki-ci
+on:
+  pull_request:
+    paths:
+      - "production/helm/loki/**"
+
+jobs:
+  publish-diff:
+    name: Publish Rendered Helm Chart Diff
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Helm
+        uses: azure/setup-helm@v4
+
+      - name: Add required Helm repositories
+        run: |
+          helm repo add minio https://charts.min.io/
+          helm repo add grafana https://grafana.github.io/helm-charts
+          helm repo add grafana-operator https://grafana.github.io/helm-charts
+          helm repo update
+
+      - name: Prepare directories for base and PR branches
+        run: |
+          mkdir -p ${{ github.workspace }}/base
+          mkdir -p ${{ github.workspace }}/pr
+          mkdir -p ${{ github.workspace }}/output
+          mkdir -p ${{ github.workspace }}/output/base
+          mkdir -p ${{ github.workspace }}/output/pr
+
+      - name: Checkout base branch to 'base' folder within workspace
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+          path: ${{ github.workspace }}/base
+
+      - name: Checkout PR branch to 'pr' folder within workspace
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          path: ${{ github.workspace }}/pr
+
+      - name: Render Helm chart for each scenario in the base branch
+        run: |
+          cd ${{ github.workspace }}/base/production/helm/loki
+          if [ ! -d "scenarios" ]; then
+            echo "Directory with the scenarios does not exist in base branch, skipping rendering them."
+            exit 0
+          fi
+          helm dependency build
+          for file in scenarios/*.yaml; do
+            cat "$file"
+            schenario_folder=${{ github.workspace }}/output/base/$(basename $file .yaml)
+            mkdir $schenario_folder
+            helm template loki-test-chart-name . -f $file --output-dir $schenario_folder
+          done
+
+      - name: Render Helm chart for each scenario in the PR branch
+        run: |
+          cd ${{ github.workspace }}/pr/production/helm/loki
+          helm dependency build
+          for file in scenarios/*.yaml; do
+            cat "$file"
+            schenario_folder=${{ github.workspace }}/output/pr/$(basename $file .yaml)
+            mkdir $schenario_folder
+            helm template loki-test-chart-name . -f $file --output-dir $schenario_folder
+          done
+
+      - name: Calculate the diff between base and PR rendered manifests for each scenario
+        run: |
+          cd ${{ github.workspace }}/pr/production/helm/loki
+          for scenario_file in scenarios/*.yaml; do
+            added_files='[]'
+            modified_files='[]'
+            removed_files='[]'
+
+            scenario_name=$(basename $scenario_file .yaml)
+            base_branch_dir=${{ github.workspace }}/output/base/$scenario_name
+            pr_branch_dir=${{ github.workspace }}/output/pr/$scenario_name
+
+            echo "Comparing directories: $base_branch_dir and $pr_branch_dir"
+
+            # Find all files in the left and right directories
+            base_branch_files=$(if [[ -d "$base_branch_dir" ]]; then find "$base_branch_dir" -type f | sed "s|$base_branch_dir/||"; else echo ""; fi)
+            pr_branch_files=$(find "$pr_branch_dir" -type f | sed "s|$pr_branch_dir/||")
+
+            # Check for modified and removed files
+            for file in $base_branch_files; do
+              echo "check if file exists: $file"
+              if [[ -f "$pr_branch_dir/$file" ]]; then
+                echo "File exists in both directories, check if it is modified"
+                if ! diff -q "$base_branch_dir/$file" "$pr_branch_dir/$file" >/dev/null; then
+                  echo "file is modified $file"
+                  file_diff=$(diff -c "$base_branch_dir/$file" "$pr_branch_dir/$file" || true)
+                  diff_obj=$(jq -n --arg file "$file" --arg diff "$file_diff" '{"filename": $file, "diff": $diff}')
+                  # Append the new object to the JSON array using jq
+                  modified_files=$(echo "$modified_files" | jq --argjson diff_obj "$diff_obj" '. += [$diff_obj]')
+                else
+                  echo "file is not modified"
+                fi
+              else
+                echo "file is removed $file"
+                # File is missing in the PR directory
+                file_content=$(cat "$base_branch_dir/$file")
+                removed_obj=$(jq -n --arg filename "$file" --arg content "$file_content" '{"filename": $filename, "content": $content}')
+                # Append the new object to the JSON array using jq
+                removed_files=$(echo "$removed_files" | jq --argjson removed_obj "$removed_obj" '. += [$removed_obj]')
+              fi
+            done
+
+            # Check for added files in the right directory
+            for file in $pr_branch_files; do
+              if [[ ! -f "$base_branch_dir/$file" ]]; then
+                echo "added file detected"
+
+                # File is missing in the PR directory
+                file_content=$(cat "$pr_branch_dir/$file")
+                added_obj=$(jq -n --arg file "$file" --arg content "$file_content" '{"filename": $file, "content": $content}')
+                # Append the new object to the JSON array using jq
+                added_files=$(echo "$added_files" | jq --argjson added_obj "$added_obj" '. += [$added_obj]')
+              fi
+            done
+          
+            scenario_output_dir="${{ github.workspace }}/output/$scenario_name"
+            mkdir $scenario_output_dir
+            echo $added_files > $scenario_output_dir/added_files.json
+            echo $modified_files > $scenario_output_dir/modified_files.json
+            echo $removed_files > $scenario_output_dir/removed_files.json
+            echo $removed_files
+          done
+
+      - name: Generate Markdown Summary
+        run: |
+          # Initialize the Markdown output file
+          output_file="${{ github.workspace }}/output/diff_summary.md"
+          echo "# Kubernetes Manifest Diff Summary" > $output_file
+
+          # Iterate over each scenario file
+          for file in ${{ github.workspace }}/pr/production/helm/loki/scenarios/*.yaml; do
+            scenario=$(basename "$file" .yaml)
+            echo "Processing scenario: $scenario"
+          
+            # Read JSON data for added, modified, and removed files
+            added_files=$(cat ${{ github.workspace }}/output/$scenario/added_files.json)
+            modified_files=$(cat ${{ github.workspace }}/output/$scenario/modified_files.json)
+            removed_files=$(cat ${{ github.workspace }}/output/$scenario/removed_files.json)
+          
+            # Count the number of added, modified, and removed files
+            num_added=$(echo "$added_files" | jq length)
+            num_modified=$(echo "$modified_files" | jq length)
+            num_removed=$(echo "$removed_files" | jq length)
+  
+            # Create a header for the scenario
+            echo -e "\n<details><summary>Scenario: $scenario (Added: $num_added, Modified: $num_modified, Removed: $num_removed) </summary>\n" >> $output_file
+            echo -e "<p>\n\n" >> $output_file
+
+            # Add summary counts
+            echo -e "\n**Summary:**" >> $output_file
+            echo -e "\n- **Added:** $num_added" >> $output_file
+            echo -e "\n- **Modified:** $num_modified" >> $output_file
+            echo -e "\n- **Removed:** $num_removed" >> $output_file
+
+            # Add details for added files
+            echo -e "\n### Added Files" >> $output_file
+            if [[ "$num_added" -gt 0 ]]; then
+             echo "$added_files" | jq -c '.[]' | while read -r obj; do
+               filename=$(echo "$obj" | jq -r '.filename')
+               content=$(echo "$obj" | jq -r '.content')
+               echo -e "\n<details><summary>$filename</summary>" >> $output_file
+               echo -e "\n\`\`\`yaml\n$content\n\`\`\`\n</details>" >> $output_file
+             done
+            else
+             echo -e "\n_No added files_\n" >> $output_file
+            fi
+
+            # Add details for modified files
+            echo -e "\n### Modified Files" >> $output_file
+            if [[ "$num_modified" -gt 0 ]]; then
+             echo "$modified_files" | jq -c '.[]' | while read -r obj; do
+               filename=$(echo "$obj" | jq -r '.filename')
+               diff=$(echo "$obj" | jq -r '.diff')
+               echo -e "\n<details><summary>$filename</summary>" >> $output_file
+               echo -e "\n\`\`\`diff\n$diff\n\`\`\`\n</details>" >> $output_file
+             done
+            else
+             echo -e "\n_No modified files_\n" >> $output_file
+            fi
+
+            # Add details for removed files
+            echo -e "\n### Removed Files" >> $output_file
+            if [[ "$num_removed" -gt 0 ]]; then
+             echo "$removed_files" | jq -c '.[]' | while read -r obj; do
+               filename=$(echo "$obj" | jq -r '.filename')
+               content=$(echo "$obj" | jq -r '.content')
+               echo -e "\n<details><summary>$filename</summary>" >> $output_file
+               echo -e "\n\`\`\`yaml\n$content\n\`\`\`\n</details>" >> $output_file
+             done
+            else
+             echo -e "\n_No removed files_\n" >> $output_file
+            fi
+
+            # close <p> and <details>
+            echo -e "\n\n</p>\n</details>" >> $output_file
+          done
+
+      - name: Post diff as PR comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          hide_and_recreate: true
+          hide_classify: "OUTDATED"
+          path: ${{ github.workspace }}/output/diff_summary.md

.github/workflows/metrics-collector.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   main:
-    if: github.owner == "grafana"
+    if: github.owner == 'grafana'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions

.github/workflows/syft-sbom-ci.yml

@@ -14,7 +14,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Anchore SBOM Action
-      uses: anchore/[email protected].1
+      uses: anchore/[email protected].2
       with:
          artifact-name: ${{ github.event.repository.name }}-spdx.json
 

.gitignore

@@ -60,3 +60,6 @@ nix/result
 
 # helm test
 production/helm/loki/src/helm-test/helm-test
+
+# Go telemetry
+.config

.release-please-manifest.json

@@ -1,4 +1,4 @@
 {
   ".": "3.1.1",
-  "operator": "0.6.1"
+  "operator": "0.6.2"
 }

Makefile

@@ -193,8 +193,8 @@ production/helm/loki/src/helm-test/helm-test:
 helm-lint: ## run helm linter
 	$(MAKE) -BC production/helm/loki lint
 
-helm-docs:
-	helm-docs -c production/helm/loki -g production/helm/loki
+helm-docs: ## generate reference documentation
+	$(MAKE) -BC docs sources/setup/install/helm/reference.md
 
 #################
 # Loki-QueryTee #

clients/cmd/docker-driver/http.go

@@ -56,7 +56,7 @@ func handlers(h *sdk.Handler, d *driver) {
 		respond(nil, w)
 	})
 
-	h.HandleFunc("/LogDriver.Capabilities", func(w http.ResponseWriter, r *http.Request) {
+	h.HandleFunc("/LogDriver.Capabilities", func(w http.ResponseWriter, _ *http.Request) {
 		_ = json.NewEncoder(w).Encode(&CapabilitiesResponse{
 			Cap: logger.Capability{ReadLogs: true},
 		})

clients/pkg/logentry/stages/match.go

@@ -51,10 +51,10 @@ func validateMatcherConfig(cfg *MatcherConfig) (logql.Expr, error) {
 		return nil, errors.New(ErrUnknownMatchAction)
 	}
 
-	if cfg.Action == MatchActionKeep && (cfg.Stages == nil || len(cfg.Stages) == 0) {
+	if cfg.Action == MatchActionKeep && (len(cfg.Stages) == 0) {
 		return nil, errors.New(ErrMatchRequiresStages)
 	}
-	if cfg.Action == MatchActionDrop && (cfg.Stages != nil && len(cfg.Stages) != 0) {
+	if cfg.Action == MatchActionDrop && (len(cfg.Stages) != 0) {
 		return nil, errors.New(ErrStagesWithDropLine)
 	}
 

clients/pkg/promtail/client/client_test.go

@@ -727,7 +727,7 @@ func Test_Tripperware(t *testing.T) {
 	var called bool
 	c, err := NewWithTripperware(metrics, Config{
 		URL: flagext.URLValue{URL: url},
-	}, 0, 0, false, log.NewNopLogger(), func(rt http.RoundTripper) http.RoundTripper {
+	}, 0, 0, false, log.NewNopLogger(), func(_ http.RoundTripper) http.RoundTripper {
 		return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
 			require.Equal(t, r.URL.String(), "http://foo.com")
 			called = true

clients/pkg/promtail/discovery/consulagent/consul_test.go

@@ -464,14 +464,14 @@ func TestGetDatacenterShouldReturnError(t *testing.T) {
 	}{
 		{
 			// Define a handler that will return status 500.
-			handler: func(w http.ResponseWriter, r *http.Request) {
+			handler: func(w http.ResponseWriter, _ *http.Request) {
 				w.WriteHeader(500)
 			},
 			errMessage: "Unexpected response code: 500 ()",
 		},
 		{
 			// Define a handler that will return incorrect response.
-			handler: func(w http.ResponseWriter, r *http.Request) {
+			handler: func(w http.ResponseWriter, _ *http.Request) {
 				_, err := w.Write([]byte(`{"Config": {"Not-Datacenter": "test-dc"}}`))
 				require.NoError(t, err)
 			},

clients/pkg/promtail/targets/cloudflare/target_test.go

@@ -65,7 +65,7 @@ func Test_CloudflareTarget(t *testing.T) {
 		logs: []string{},
 	}, nil)
 	// replace the client.
-	getClient = func(apiKey, zoneID string, fields []string) (Client, error) {
+	getClient = func(_, _ string, _ []string) (Client, error) {
 		return cfClient, nil
 	}
 
@@ -114,7 +114,7 @@ func Test_RetryErrorLogpullReceived(t *testing.T) {
 		err: ErrorLogpullReceived,
 	}, nil).Times(2) // just retry once
 	// replace the client
-	getClient = func(apiKey, zoneID string, fields []string) (Client, error) {
+	getClient = func(_, _ string, _ []string) (Client, error) {
 		return cfClient, nil
 	}
 	defaultBackoff.MinBackoff = 0
@@ -159,7 +159,7 @@ func Test_RetryErrorIterating(t *testing.T) {
 		err: ErrorLogpullReceived,
 	}, nil).Once()
 	// replace the client.
-	getClient = func(apiKey, zoneID string, fields []string) (Client, error) {
+	getClient = func(_, _ string, _ []string) (Client, error) {
 		return cfClient, nil
 	}
 	// retries as fast as possible.
@@ -210,7 +210,7 @@ func Test_CloudflareTargetError(t *testing.T) {
 	// setup errors for all retries
 	cfClient.On("LogpullReceived", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("no logs"))
 	// replace the client.
-	getClient = func(apiKey, zoneID string, fields []string) (Client, error) {
+	getClient = func(_, _ string, _ []string) (Client, error) {
 		return cfClient, nil
 	}
 
@@ -263,7 +263,7 @@ func Test_CloudflareTargetError168h(t *testing.T) {
 	// setup errors for all retries
 	cfClient.On("LogpullReceived", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("HTTP status 400: bad query: error parsing time: invalid time range: too early: logs older than 168h0m0s are not available"))
 	// replace the client.
-	getClient = func(apiKey, zoneID string, fields []string) (Client, error) {
+	getClient = func(_, _ string, _ []string) (Client, error) {
 		return cfClient, nil
 	}
 

clients/pkg/promtail/targets/gcplog/pull_target.go

@@ -127,7 +127,7 @@ func (t *pullTarget) consumeSubscription() {
 	defer t.cancel()
 
 	for t.backoff.Ongoing() {
-		err := t.sub.Receive(t.ctx, func(ctx context.Context, m *pubsub.Message) {
+		err := t.sub.Receive(t.ctx, func(_ context.Context, m *pubsub.Message) {
 			t.msgs <- m
 			t.backoff.Reset()
 		})

clients/pkg/promtail/targets/journal/journaltarget_test.go

@@ -45,7 +45,7 @@ func (r *mockJournalReader) Follow(until <-chan time.Time, _ io.Writer) error {
 }
 
 func newMockJournalEntry(entry *sdjournal.JournalEntry) journalEntryFunc {
-	return func(c sdjournal.JournalReaderConfig, cursor string) (*sdjournal.JournalEntry, error) {
+	return func(_ sdjournal.JournalReaderConfig, _ string) (*sdjournal.JournalEntry, error) {
 		return entry, nil
 	}
 }