Merge remote-tracking branch 'origin/main' into remove-affinity-defau…

…lt-in-sb-sts-to-bypass-object-merge-problem

.github/release-workflows.jsonnet

@@ -70,7 +70,7 @@ local imagePrefix = 'grafana';
       imagePrefix='grafana',
       releaseLibRef=releaseLibRef,
       releaseRepo='grafana/loki',
-      useGitHubAppToken=false,
+      useGitHubAppToken=true,
     ), false, false
   ),
   'check.yml': std.manifestYamlDoc({

.github/workflows/add-to-docs-project.yml

@@ -0,0 +1,15 @@
+name: Add to docs project
+on:
+  issues:
+    types: [labeled]
+  pull_request:
+    types: [labeled]
+jobs:
+  main:
+    if: ${{ github.event.label.name == 'type/docs' }}
+    permissions:
+      contents: read
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: grafana/writers-toolkit/add-to-docs-project@add-to-docs-project/v1

.github/workflows/backport.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   main:
+    if: github.repository == 'grafana/loki'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions
@@ -17,10 +18,17 @@ jobs:
           ref: main
       - name: Install Actions
         run: npm install --production --prefix ./actions
+      - id: "get_github_app_token"
+        name: "get github app token"
+        uses: "actions/create-github-app-token@v1"
+        with:
+          app-id: "${{ secrets.APP_ID }}"
+          owner: "${{ github.repository_owner }}"
+          private-key: "${{ secrets.APP_PRIVATE_KEY }}"
       - name: Run backport
         uses: ./actions/backport
         with:
           metricsWriteAPIKey: ${{secrets.GRAFANA_MISC_STATS_API_KEY}}
-          token: ${{secrets.GH_BOT_ACCESS_TOKEN}}
+          token: ${{ steps.get_github_app_token.outputs.token }}
           labelsToAdd: "backport"
           title: "chore: [{{base}}] {{originalTitle}}"

.github/workflows/helm-loki-ci.yml

@@ -0,0 +1,215 @@
+---
+name: helm-loki-ci
+on:
+  pull_request:
+    paths:
+      - "production/helm/loki/**"
+
+jobs:
+  publish-diff:
+    # temporarily disable the workflow for the PRs where PRs branch is from fork.
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    name: Publish Rendered Helm Chart Diff
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Helm
+        uses: azure/setup-helm@v4
+
+      - name: Add required Helm repositories
+        run: |
+          helm repo add minio https://charts.min.io/
+          helm repo add grafana https://grafana.github.io/helm-charts
+          helm repo add grafana-operator https://grafana.github.io/helm-charts
+          helm repo update
+
+      - name: Prepare directories for base and PR branches
+        run: |
+          mkdir -p ${{ github.workspace }}/base
+          mkdir -p ${{ github.workspace }}/pr
+          mkdir -p ${{ github.workspace }}/output
+          mkdir -p ${{ github.workspace }}/output/base
+          mkdir -p ${{ github.workspace }}/output/pr
+
+      - name: Checkout base branch to 'base' folder within workspace
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+          repository: ${{ github.event.pull_request.base.repo.full_name }}
+          path: ${{ github.workspace }}/base
+
+      - name: Checkout PR branch to 'pr' folder within workspace
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          path: ${{ github.workspace }}/pr
+
+      - name: Render Helm chart for each scenario in the base branch
+        run: |
+          cd ${{ github.workspace }}/base/production/helm/loki
+          if [ ! -d "scenarios" ]; then
+            echo "Directory with the scenarios does not exist in base branch, skipping rendering them."
+            exit 0
+          fi
+          helm dependency build
+          for file in scenarios/*.yaml; do
+            cat "$file"
+            schenario_folder=${{ github.workspace }}/output/base/$(basename $file .yaml)
+            mkdir $schenario_folder
+            helm template loki-test-chart-name . -f $file --output-dir $schenario_folder
+          done
+
+      - name: Render Helm chart for each scenario in the PR branch
+        run: |
+          cd ${{ github.workspace }}/pr/production/helm/loki
+          helm dependency build
+          for file in scenarios/*.yaml; do
+            cat "$file"
+            schenario_folder=${{ github.workspace }}/output/pr/$(basename $file .yaml)
+            mkdir $schenario_folder
+            helm template loki-test-chart-name . -f $file --output-dir $schenario_folder
+          done
+
+      - name: Calculate the diff between base and PR rendered manifests for each scenario
+        run: |
+          cd ${{ github.workspace }}/pr/production/helm/loki
+          for scenario_file in scenarios/*.yaml; do
+            added_files='[]'
+            modified_files='[]'
+            removed_files='[]'
+
+            scenario_name=$(basename $scenario_file .yaml)
+            base_branch_dir=${{ github.workspace }}/output/base/$scenario_name
+            pr_branch_dir=${{ github.workspace }}/output/pr/$scenario_name
+
+            echo "Comparing directories: $base_branch_dir and $pr_branch_dir"
+
+            # Find all files in the left and right directories
+            base_branch_files=$(if [[ -d "$base_branch_dir" ]]; then find "$base_branch_dir" -type f | sed "s|$base_branch_dir/||"; else echo ""; fi)
+            pr_branch_files=$(find "$pr_branch_dir" -type f | sed "s|$pr_branch_dir/||")
+
+            # Check for modified and removed files
+            for file in $base_branch_files; do
+              echo "check if file exists: $file"
+              if [[ -f "$pr_branch_dir/$file" ]]; then
+                echo "File exists in both directories, check if it is modified"
+                if ! diff -q "$base_branch_dir/$file" "$pr_branch_dir/$file" >/dev/null; then
+                  echo "file is modified $file"
+                  file_diff=$(diff -c "$base_branch_dir/$file" "$pr_branch_dir/$file" || true)
+                  diff_obj=$(jq -n --arg file "$file" --arg diff "$file_diff" '{"filename": $file, "diff": $diff}')
+                  # Append the new object to the JSON array using jq
+                  modified_files=$(echo "$modified_files" | jq --argjson diff_obj "$diff_obj" '. += [$diff_obj]')
+                else
+                  echo "file is not modified"
+                fi
+              else
+                echo "file is removed $file"
+                # File is missing in the PR directory
+                file_content=$(cat "$base_branch_dir/$file")
+                removed_obj=$(jq -n --arg filename "$file" --arg content "$file_content" '{"filename": $filename, "content": $content}')
+                # Append the new object to the JSON array using jq
+                removed_files=$(echo "$removed_files" | jq --argjson removed_obj "$removed_obj" '. += [$removed_obj]')
+              fi
+            done
+
+            # Check for added files in the right directory
+            for file in $pr_branch_files; do
+              if [[ ! -f "$base_branch_dir/$file" ]]; then
+                echo "added file detected"
+
+                # File is missing in the PR directory
+                file_content=$(cat "$pr_branch_dir/$file")
+                added_obj=$(jq -n --arg file "$file" --arg content "$file_content" '{"filename": $file, "content": $content}')
+                # Append the new object to the JSON array using jq
+                added_files=$(echo "$added_files" | jq --argjson added_obj "$added_obj" '. += [$added_obj]')
+              fi
+            done
+          
+            scenario_output_dir="${{ github.workspace }}/output/$scenario_name"
+            mkdir $scenario_output_dir
+            echo $added_files > $scenario_output_dir/added_files.json
+            echo $modified_files > $scenario_output_dir/modified_files.json
+            echo $removed_files > $scenario_output_dir/removed_files.json
+            echo $removed_files
+          done
+
+      - name: Generate Markdown Summary
+        run: |
+          # Initialize the Markdown output file
+          output_file="${{ github.workspace }}/output/diff_summary.md"
+          echo "# Kubernetes Manifest Diff Summary" > $output_file
+
+          # Iterate over each scenario file
+          for file in ${{ github.workspace }}/pr/production/helm/loki/scenarios/*.yaml; do
+            scenario=$(basename "$file" .yaml)
+            echo "Processing scenario: $scenario"
+          
+            # Read JSON data for added, modified, and removed files
+            added_files=$(cat ${{ github.workspace }}/output/$scenario/added_files.json)
+            modified_files=$(cat ${{ github.workspace }}/output/$scenario/modified_files.json)
+            removed_files=$(cat ${{ github.workspace }}/output/$scenario/removed_files.json)
+          
+            # Count the number of added, modified, and removed files
+            num_added=$(echo "$added_files" | jq length)
+            num_modified=$(echo "$modified_files" | jq length)
+            num_removed=$(echo "$removed_files" | jq length)
+  
+            # Create a header for the scenario
+            echo -e "\n<details><summary>Scenario: $scenario (Added: $num_added, Modified: $num_modified, Removed: $num_removed) </summary>\n" >> $output_file
+            echo -e "<p>\n\n" >> $output_file
+
+            # Add summary counts
+            echo -e "\n**Summary:**" >> $output_file
+            echo -e "\n- **Added:** $num_added" >> $output_file
+            echo -e "\n- **Modified:** $num_modified" >> $output_file
+            echo -e "\n- **Removed:** $num_removed" >> $output_file
+
+            # Add details for added files
+            echo -e "\n### Added Files" >> $output_file
+            if [[ "$num_added" -gt 0 ]]; then
+             echo "$added_files" | jq -c '.[]' | while read -r obj; do
+               filename=$(echo "$obj" | jq -r '.filename')
+               content=$(echo "$obj" | jq -r '.content')
+               echo -e "\n<details><summary>$filename</summary>" >> $output_file
+               echo -e "\n\`\`\`yaml\n$content\n\`\`\`\n</details>" >> $output_file
+             done
+            else
+             echo -e "\n_No added files_\n" >> $output_file
+            fi
+
+            # Add details for modified files
+            echo -e "\n### Modified Files" >> $output_file
+            if [[ "$num_modified" -gt 0 ]]; then
+             echo "$modified_files" | jq -c '.[]' | while read -r obj; do
+               filename=$(echo "$obj" | jq -r '.filename')
+               diff=$(echo "$obj" | jq -r '.diff')
+               echo -e "\n<details><summary>$filename</summary>" >> $output_file
+               echo -e "\n\`\`\`diff\n$diff\n\`\`\`\n</details>" >> $output_file
+             done
+            else
+             echo -e "\n_No modified files_\n" >> $output_file
+            fi
+
+            # Add details for removed files
+            echo -e "\n### Removed Files" >> $output_file
+            if [[ "$num_removed" -gt 0 ]]; then
+             echo "$removed_files" | jq -c '.[]' | while read -r obj; do
+               filename=$(echo "$obj" | jq -r '.filename')
+               content=$(echo "$obj" | jq -r '.content')
+               echo -e "\n<details><summary>$filename</summary>" >> $output_file
+               echo -e "\n\`\`\`yaml\n$content\n\`\`\`\n</details>" >> $output_file
+             done
+            else
+             echo -e "\n_No removed files_\n" >> $output_file
+            fi
+
+            # close <p> and <details>
+            echo -e "\n\n</p>\n</details>" >> $output_file
+          done
+
+      - name: Post diff as PR comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          hide_and_recreate: true
+          hide_classify: "OUTDATED"
+          path: ${{ github.workspace }}/output/diff_summary.md

.github/workflows/metrics-collector.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   main:
-    if: github.owner == "grafana"
+    if: github.owner == 'grafana'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions

.github/workflows/publish-technical-documentation-next.yml

@@ -1,37 +1,21 @@
-name: "publish-technical-documentation-next"
+name: publish-technical-documentation-next
 
 on:
   push:
     branches:
-      - "main"
+      - main
     paths:
       - "docs/sources/**"
   workflow_dispatch:
 jobs:
   sync:
-    runs-on: "ubuntu-latest"
+    if: github.repository == 'grafana/loki'
+    permissions:
+      contents: read
+      id-token: write
+    runs-on: ubuntu-latest
     steps:
-      - name: "Check out code"
-        uses: "actions/checkout@v4"
-
-      - name: "Clone website-sync Action"
-        # WEBSITE_SYNC_TOKEN is a fine-grained GitHub Personal Access Token that expires.
-        # It must be regenerated in the grafanabot GitHub account and requires a Grafana organization
-        # GitHub administrator to update the organization secret.
-        # The IT helpdesk can update the organization secret.
-        run: "git clone --single-branch --no-tags --depth 1 -b master https://grafanabot:${{ secrets.WEBSITE_SYNC_TOKEN }}@github.com/grafana/website-sync ./.github/actions/website-sync"
-
-      - name: "Publish to website repository (next)"
-        uses: "./.github/actions/website-sync"
-        id: "publish-next"
+      - uses: actions/checkout@v4
+      - uses: grafana/writers-toolkit/publish-technical-documentation@publish-technical-documentation/v1
         with:
-          repository: "grafana/website"
-          branch: "master"
-          host: "github.com"
-          # PUBLISH_TO_WEBSITE_TOKEN is a fine-grained GitHub Personal Access Token that expires.
-          # It must be regenerated in the grafanabot GitHub account and requires a Grafana organization
-          # GitHub administrator to update the organization secret.
-          # The IT helpdesk can update the organization secret.
-          github_pat: "grafanabot:${{ secrets.PUBLISH_TO_WEBSITE_TOKEN }}"
-          source_folder: "docs/sources"
-          target_folder: "content/docs/loki/next"
+          website_directory: content/docs/loki/next