From 9279832610c312d81c481499ce3911d6004c6a42 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Thu, 5 Dec 2024 13:08:40 -0500
Subject: [PATCH 01/14] chore(deps): update grafana/promtail docker tag to
 v3.3.1 (#15285)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Paul Rogers <paul.rogers@grafana.com>
---
 docs/sources/send-data/promtail/cloud/eks/values.yaml | 2 +-
 nix/packages/loki.nix                                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sources/send-data/promtail/cloud/eks/values.yaml b/docs/sources/send-data/promtail/cloud/eks/values.yaml
index 8d8fcb7b87d23..c2f26f4de1ec8 100644
--- a/docs/sources/send-data/promtail/cloud/eks/values.yaml
+++ b/docs/sources/send-data/promtail/cloud/eks/values.yaml
@@ -17,7 +17,7 @@ initContainer:
 
 image:
   repository: grafana/promtail
-  tag: 3.3.0
+  tag: 3.3.1
   pullPolicy: IfNotPresent
   ## Optionally specify an array of imagePullSecrets.
   ## Secrets must be manually created in the namespace.
diff --git a/nix/packages/loki.nix b/nix/packages/loki.nix
index a4f5b4b9feb28..7da43f05d3ede 100644
--- a/nix/packages/loki.nix
+++ b/nix/packages/loki.nix
@@ -5,7 +5,7 @@ let
     pname = "lambda-promtail";
 
     src = ./../../tools/lambda-promtail;
-    vendorHash = "sha256-qsgaWkuSVo5Clb9FpS4SUIHVZZjpfjv0dKtwSUIX7a8=";
+    vendorHash = "sha256-yQIRFUhod91HiPS5IKm7eNeIXJzBWVcvIXf9qMncTKw=";
 
     doCheck = false;
 

From 429c3de6490cf9299d6b07f42573842ab0f14620 Mon Sep 17 00:00:00 2001
From: J Stickler <julie.stickler@grafana.com>
Date: Thu, 5 Dec 2024 15:33:56 -0500
Subject: [PATCH 02/14] docs: Release notes for 2.9.11 (#15280)

---
 docs/sources/release-notes/v2-9.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/sources/release-notes/v2-9.md b/docs/sources/release-notes/v2-9.md
index 3c113c53b3f3f..6be5e268e576a 100644
--- a/docs/sources/release-notes/v2-9.md
+++ b/docs/sources/release-notes/v2-9.md
@@ -36,6 +36,13 @@ For a full list of all changes and fixes, refer to the [CHANGELOG](https://githu
 
 ## Bug fixes
 
+### 2.9.11 (2026-12-05)
+
+- **docker:** Update Docker to 23.0.15 ([#](https://github.com/grafana/loki/issues/)).
+- **lamba-promtail:** Lamba-promtail updates, some of which address CVEs([#14105](https://github.com/grafana/loki/issues/14105)).
+- **promtail:** Switch Promtail base image from Debian to Ubuntu to fix critical security issues ([#15195](https://github.com/grafana/loki/issues/15195)).
+- **storage:** Fix bug in cache of the index object client([#10585](https://github.com/grafana/loki/issues/10585)).
+
 ### 2.9.10 (2026-08-09)
 
 - Update dependencies versions to remove CVE ([#13835](https://github.com/grafana/loki/pull/13835)) ([567bef2](https://github.com/grafana/loki/commit/567bef286376663407c54f5da07fa00963ba5485)).

From ed0b1f9edc04dc127a9ab02d6c7cf7f869ef34bb Mon Sep 17 00:00:00 2001
From: "loki-gh-app[bot]" <160051081+loki-gh-app[bot]@users.noreply.github.com>
Date: Thu, 5 Dec 2024 14:44:56 -0700
Subject: [PATCH 03/14] chore: release loki helm chart 6.23.0 (#15271)

Co-authored-by: loki-gh-app[bot] <loki-gh-app[bot]@users.noreply.github.com>
Co-authored-by: Trevor Whitney <trevorjwhitney@gmail.com>
---
 docs/sources/setup/install/helm/reference.md | 6 +++---
 production/helm/loki/CHANGELOG.md            | 8 +++++++-
 production/helm/loki/Chart.yaml              | 4 ++--
 production/helm/loki/README.md               | 2 +-
 production/helm/loki/values.yaml             | 4 ++--
 5 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md
index 4975387e96dc8..0e5b534e10851 100644
--- a/docs/sources/setup/install/helm/reference.md
+++ b/docs/sources/setup/install/helm/reference.md
@@ -3062,7 +3062,7 @@ null
     "pullPolicy": "IfNotPresent",
     "registry": "docker.io",
     "repository": "grafana/enterprise-logs",
-    "tag": "3.3.0"
+    "tag": "3.3.1"
   },
   "license": {
     "contents": "NOTAVALIDLICENSE"
@@ -3235,7 +3235,7 @@ null
 			<td>string</td>
 			<td>Docker image tag</td>
 			<td><pre lang="json">
-"3.3.0"
+"3.3.1"
 </pre>
 </td>
 		</tr>
@@ -6090,7 +6090,7 @@ null
 			<td>string</td>
 			<td>Overrides the image tag whose default is the chart's appVersion</td>
 			<td><pre lang="json">
-"3.3.0"
+"3.3.1"
 </pre>
 </td>
 		</tr>
diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md
index 9f21593b2e403..dfc51d67d0a42 100644
--- a/production/helm/loki/CHANGELOG.md
+++ b/production/helm/loki/CHANGELOG.md
@@ -12,11 +12,17 @@ Entries should be ordered as follows:
 Entries should include a reference to the pull request that introduced the change.
 
 [//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)
-- [CHANGE] Changed version of Minio helm chart to 5.3.0 (#14834)
 
+## 6.23.0
+
+- [CHANGE] Changed version of Grafana Loki to 3.3.1
+- [CHANGE] Changed version of Minio helm chart to 5.3.0 (#14834)
 - [BUGFIX] Add default wal dir to ruler config ([#14920](https://github.com/grafana/loki/pull/14920))
+
 ## 6.22.0
 
+## 6.21.0
+
 ## 6.20.0
 
 - [CHANGE] Changed version of Grafana Loki to 3.3.0
diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml
index 7fd96976ad7ad..e1b62848c2e0b 100644
--- a/production/helm/loki/Chart.yaml
+++ b/production/helm/loki/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: loki
 description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes.
 type: application
-appVersion: 3.3.0
-version: 6.22.0
+appVersion: 3.3.1
+version: 6.23.0
 home: https://grafana.github.io/helm-charts
 sources:
   - https://github.com/grafana/loki
diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md
index 7396e9c3a61e6..bbc39ed7c4bef 100644
--- a/production/helm/loki/README.md
+++ b/production/helm/loki/README.md
@@ -1,6 +1,6 @@
 # loki
 
-![Version: 6.22.0](https://img.shields.io/badge/Version-6.22.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.3.0](https://img.shields.io/badge/AppVersion-3.3.0-informational?style=flat-square)
+![Version: 6.23.0](https://img.shields.io/badge/Version-6.23.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.3.1](https://img.shields.io/badge/AppVersion-3.3.1-informational?style=flat-square)
 
 Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes.
 
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
index 94afe8f5a829e..855e589a097cb 100644
--- a/production/helm/loki/values.yaml
+++ b/production/helm/loki/values.yaml
@@ -56,7 +56,7 @@ loki:
     # -- Docker image repository
     repository: grafana/loki
     # -- Overrides the image tag whose default is the chart's appVersion
-    tag: 3.3.0
+    tag: 3.3.1
     # -- Overrides the image tag with an image digest
     digest: null
     # -- Docker image pull policy
@@ -520,7 +520,7 @@ enterprise:
     # -- Docker image repository
     repository: grafana/enterprise-logs
     # -- Docker image tag
-    tag: 3.3.0
+    tag: 3.3.1
     # -- Overrides the image tag with an image digest
     digest: null
     # -- Docker image pull policy

From 7a7b3b1ee3c73d4ed7c802becb98ec795e042572 Mon Sep 17 00:00:00 2001
From: Shantanu Alshi <shantanu.alshi@grafana.com>
Date: Fri, 6 Dec 2024 15:25:22 +0530
Subject: [PATCH 04/14] fix(aggregated_metrics): Fix the IsError method causing
 retries (#15296)

Co-authored-by: Ned Andreev <n@andreev.sh>
---
 pkg/util/http.go      |  2 +-
 pkg/util/http_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/pkg/util/http.go b/pkg/util/http.go
index 2d9b8a7e29039..bd154c1a5f215 100644
--- a/pkg/util/http.go
+++ b/pkg/util/http.go
@@ -329,7 +329,7 @@ func ErrorTypeFromHTTPStatus(status int) string {
 }
 
 func IsError(status int) bool {
-	return status/200 != 0
+	return status < 200 || status >= 300
 }
 
 func IsServerError(status int) bool {
diff --git a/pkg/util/http_test.go b/pkg/util/http_test.go
index c41ba0eb1525d..fce7b0ab490ec 100644
--- a/pkg/util/http_test.go
+++ b/pkg/util/http_test.go
@@ -280,3 +280,59 @@ func TestErrorTypeFromHTTPStatus(t *testing.T) {
 		})
 	}
 }
+
+func TestIsError(t *testing.T) {
+	tests := []struct {
+		name           string
+		status         int
+		expectedResult bool
+	}{
+		{
+			name:           "200 OK",
+			status:         200,
+			expectedResult: false,
+		},
+		{
+			name:           "201 Created",
+			status:         201,
+			expectedResult: false,
+		},
+		{
+			name:           "400 Bad Request",
+			status:         400,
+			expectedResult: true,
+		},
+		{
+			name:           "404 Not Found",
+			status:         404,
+			expectedResult: true,
+		},
+		{
+			name:           "429 Too Many Requests",
+			status:         429,
+			expectedResult: true,
+		},
+		{
+			name:           "500 Internal Server Error",
+			status:         500,
+			expectedResult: true,
+		},
+		{
+			name:           "503 Service Unavailable",
+			status:         503,
+			expectedResult: true,
+		},
+		{
+			name:           "600 Unknown",
+			status:         600,
+			expectedResult: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := util.IsError(tt.status)
+			assert.Equal(t, tt.expectedResult, result)
+		})
+	}
+}

From 079779e8ba507fa12a8fcef5e507e56d98c986fc Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Fri, 6 Dec 2024 15:06:05 +0100
Subject: [PATCH 05/14] chore: Improve containsLower performance using quick
 rejection (#15076)

---
 pkg/logql/log/filter.go      | 77 +++++++++++++++++++---------
 pkg/logql/log/filter_test.go | 99 +++++++++++++++++++++++++++++++++++-
 2 files changed, 151 insertions(+), 25 deletions(-)

diff --git a/pkg/logql/log/filter.go b/pkg/logql/log/filter.go
index dbe5c5e99ce29..03a53b92b93c4 100644
--- a/pkg/logql/log/filter.go
+++ b/pkg/logql/log/filter.go
@@ -440,6 +440,8 @@ func contains(line, substr []byte, caseInsensitive bool) bool {
 	return containsLower(line, substr)
 }
 
+// containsLower verifies if substr is a substring of line, with case insensitive comparison.
+// substr is expected to be in lowercase.
 func containsLower(line, substr []byte) bool {
 	if len(substr) == 0 {
 		return true
@@ -447,35 +449,62 @@ func containsLower(line, substr []byte) bool {
 	if len(substr) > len(line) {
 		return false
 	}
-	j := 0
-	for len(line) > 0 {
-		// ascii fast case
-		if c := line[0]; c < utf8.RuneSelf && substr[j] < utf8.RuneSelf {
-			if c == substr[j] || c+'a'-'A' == substr[j] || c == substr[j]+'a'-'A' {
-				j++
-				if j == len(substr) {
-					return true
+
+	// Fast path - try to find first byte of substr
+	firstByte := substr[0]
+	maxIndex := len(line) - len(substr)
+
+	i := 0
+	for i <= maxIndex {
+		// Find potential first byte match
+		c := line[i]
+		if c != firstByte && c+'a'-'A' != firstByte && c != firstByte+'a'-'A' {
+			i++
+			continue
+		}
+
+		// Found potential match, check rest of substr
+		matched := true
+		linePos := i
+		substrPos := 0
+
+		for linePos < len(line) && substrPos < len(substr) {
+			c := line[linePos]
+			s := substr[substrPos]
+
+			// Fast ASCII comparison
+			if c < utf8.RuneSelf && s < utf8.RuneSelf {
+				if c != s && c+'a'-'A' != s && c != s+'a'-'A' {
+					matched = false
+					break
 				}
-				line = line[1:]
+				linePos++
+				substrPos++
 				continue
 			}
-			line = line[1:]
-			j = 0
-			continue
-		}
-		// unicode slow case
-		lr, lwid := utf8.DecodeRune(line)
-		mr, mwid := utf8.DecodeRune(substr[j:])
-		if lr == mr || mr == unicode.To(unicode.LowerCase, lr) {
-			j += mwid
-			if j == len(substr) {
-				return true
+
+			// Slower Unicode path only when needed
+			lr, lineSize := utf8.DecodeRune(line[linePos:])
+			mr, substrSize := utf8.DecodeRune(substr[substrPos:])
+
+			if lr == utf8.RuneError || mr == utf8.RuneError {
+				matched = false
+				break
 			}
-			line = line[lwid:]
-			continue
+
+			if unicode.ToLower(lr) != mr {
+				matched = false
+				break
+			}
+
+			linePos += lineSize
+			substrPos += substrSize
 		}
-		line = line[lwid:]
-		j = 0
+
+		if matched && substrPos == len(substr) {
+			return true
+		}
+		i++
 	}
 	return false
 }
diff --git a/pkg/logql/log/filter_test.go b/pkg/logql/log/filter_test.go
index 9699ca2fcd44e..3568e92557cb0 100644
--- a/pkg/logql/log/filter_test.go
+++ b/pkg/logql/log/filter_test.go
@@ -112,7 +112,7 @@ func Test_SimplifiedRegex(t *testing.T) {
 			// tests all lines with both filter, they should have the same result.
 			for _, line := range fixtures {
 				l := []byte(line)
-				require.Equal(t, d.Filter(l), f.Filter(l), "regexp %s failed line: %s", test.re, line)
+				require.Equal(t, d.Filter(l), f.Filter(l), "regexp %s failed line: %s re:%v simplified:%v", test.re, line, d.Filter(l), f.Filter(l))
 			}
 		})
 	}
@@ -219,3 +219,100 @@ func benchmarkRegex(b *testing.B, re, line string, match bool) {
 func Test_rune(t *testing.T) {
 	require.True(t, newContainsFilter([]byte("foo"), true).Filter([]byte("foo")))
 }
+
+func BenchmarkContainsLower(b *testing.B) {
+	cases := []struct {
+		name     string
+		line     string
+		substr   string
+		expected bool
+	}{
+		{
+			name:     "short_line_no_match",
+			line:     "this is a short log line",
+			substr:   "missing",
+			expected: false,
+		},
+		{
+			name:     "short_line_with_match",
+			line:     "this is a short log line",
+			substr:   "SHORT",
+			expected: true,
+		},
+		{
+			name:     "long_line_no_match",
+			line:     "2023-06-14T12:34:56.789Z INFO  [service_name] This is a much longer log line with timestamps, levels and other information that typically appears in production logs. RequestID=123456 UserID=789 Action=GetUser Duration=123ms Status=200",
+			substr:   "nonexistent",
+			expected: false,
+		},
+		{
+			name:     "long_line_match_start",
+			line:     "2023-06-14T12:34:56.789Z INFO  [service_name] This is a much longer log line with timestamps, levels and other information that typically appears in production logs. RequestID=123456 UserID=789 Action=GetUser Duration=123ms Status=200",
+			substr:   "2023",
+			expected: true,
+		},
+		{
+			name:     "long_line_match_middle",
+			line:     "2023-06-14T12:34:56.789Z INFO  [service_name] This is a much longer log line with timestamps, levels and other information that typically appears in production logs. RequestID=123456 UserID=789 Action=GetUser Duration=123ms Status=200",
+			substr:   "LEVELS",
+			expected: true,
+		},
+		{
+			name:     "long_line_match_end",
+			line:     "2023-06-14T12:34:56.789Z INFO  [service_name] This is a much longer log line with timestamps, levels and other information that typically appears in production logs. RequestID=123456 UserID=789 Action=GetUser Duration=123ms Status=200",
+			substr:   "status",
+			expected: true,
+		},
+		{
+			name:     "short_unicode_line_no_match",
+			line:     "🌟 Unicode line with emojis 🎉 and special chars ñ é ß",
+			substr:   "missing",
+			expected: false,
+		},
+		{
+			name:     "short_unicode_line_with_match",
+			line:     "🌟 Unicode line with emojis 🎉 and special chars ñ é ß",
+			substr:   "EMOJIS",
+			expected: true,
+		},
+		{
+			name:     "long_unicode_line_no_match",
+			line:     "2023-06-14T12:34:56.789Z 🚀 [микросервис] Длинное сообщение с Unicode символами 统一码 が大好き! エラー分析: システムは正常に動作しています。RequestID=123456 状態=良好 Résultat=Succès ß=γ 🎯 τέλος",
+			substr:   "nonexistent",
+			expected: false,
+		},
+		{
+			name:     "long_unicode_line_match_start",
+			line:     "2023-06-14T12:34:56.789Z 🚀[МИКРОСервис] Длинное сообщение с Unicode символами 统一码 が大好き! エラー分析: システムは正常に動作しています。RequestID=123456 状態=良好 Résultat=Succès ß=γ 🎯 τέλος",
+			substr:   "микросервис",
+			expected: true,
+		},
+		{
+			name:     "long_unicode_line_match_middle",
+			line:     "2023-06-14T12:34:56.789Z 🚀 [микросервис] Длинное сообщение с Unicode символами 统一码 が大好き! エラー分析: システムは正常に動作しています。RequestID=123456 状態=良好 Résultat=Succès ß=γ 🎯 τέλος",
+			substr:   "UNICODE",
+			expected: true,
+		},
+		{
+			name:     "long_unicode_line_match_end",
+			line:     "2023-06-14T12:34:56.789Z 🚀 [микросервис] Длинное сообщение с Unicode символами 统一码 が大好き! エラー分析: システムは正常に動作しています。RequestID=123456 状態=良好 Résultat=Succès ß=γ 🎯 τέλος",
+			substr:   "τέλος",
+			expected: true,
+		},
+	}
+
+	var m bool
+	for _, c := range cases {
+		b.Run(c.name, func(b *testing.B) {
+			line := []byte(c.line)
+			substr := []byte(c.substr)
+			for i := 0; i < b.N; i++ {
+				m = containsLower(line, substr)
+			}
+			if m != c.expected {
+				b.Fatalf("expected %v but got %v", c.expected, m)
+			}
+		})
+	}
+	res = m // Avoid compiler optimization
+}

From 78d3c4415662b4003fc275b29aff053d531d90f0 Mon Sep 17 00:00:00 2001
From: Christian Haudum <christian.haudum@gmail.com>
Date: Fri, 6 Dec 2024 17:02:58 +0100
Subject: [PATCH 06/14] fix(blooms): Fix check for skipping most recent data
 when filtering blooms (#15300)

After changing the default of  `-bloom-build.planner.min-table-offset` from `1` (yesterday) to `0` (today), we noticed that the bloom gateways reported a very high percentage of missing series in blocks.

This is because the gateway received filter requests for today's block, but the requested series is too new and has not been added to the bloom block yet.

In the bloom querier on the index gateway there is a condition under which requests to the bloom gateway are skipped, because the blooms are likely not available yet.

This PR changes this condition to correctly take the min-table-offset as well as the planning-interval into account.

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>
---
 pkg/bloomgateway/querier.go | 50 ++++++++++++++++++-------------------
 pkg/loki/modules.go         |  3 ++-
 2 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/pkg/bloomgateway/querier.go b/pkg/bloomgateway/querier.go
index 472f82a1a791f..98e252b121fd2 100644
--- a/pkg/bloomgateway/querier.go
+++ b/pkg/bloomgateway/querier.go
@@ -72,8 +72,8 @@ func newQuerierMetrics(registerer prometheus.Registerer, namespace, subsystem st
 }
 
 type QuerierConfig struct {
-	// MinTableOffset is derived from the compactor's MinTableOffset
-	MinTableOffset int
+	BuildInterval    time.Duration
+	BuildTableOffset int
 }
 
 // BloomQuerier is a store-level abstraction on top of Client
@@ -119,30 +119,28 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from
 	preFilterSeries := len(grouped)
 
 	// Do not attempt to filter chunks for which there are no blooms
-	if bq.cfg.MinTableOffset > 0 {
-		minAge := truncateDay(model.Now()).Add(-1 * config.ObjectStorageIndexRequiredPeriod * time.Duration(bq.cfg.MinTableOffset-1))
-		if through.After(minAge) {
-			level.Debug(logger).Log(
-				"msg", "skip too recent chunks",
-				"tenant", tenant,
-				"from", from.Time(),
-				"through", through.Time(),
-				"responses", 0,
-				"preFilterChunks", preFilterChunks,
-				"postFilterChunks", preFilterChunks,
-				"filteredChunks", 0,
-				"preFilterSeries", preFilterSeries,
-				"postFilterSeries", preFilterSeries,
-				"filteredSeries", 0,
-			)
-
-			bq.metrics.chunksTotal.Add(float64(preFilterChunks))
-			bq.metrics.chunksFiltered.Add(0)
-			bq.metrics.seriesTotal.Add(float64(preFilterSeries))
-			bq.metrics.seriesFiltered.Add(0)
-
-			return chunkRefs, false, nil
-		}
+	minAge := model.Now().Add(-1 * (config.ObjectStorageIndexRequiredPeriod*time.Duration(bq.cfg.BuildTableOffset) + 2*bq.cfg.BuildInterval))
+	if through.After(minAge) {
+		level.Info(logger).Log(
+			"msg", "skip too recent chunks",
+			"tenant", tenant,
+			"from", from.Time(),
+			"through", through.Time(),
+			"responses", 0,
+			"preFilterChunks", preFilterChunks,
+			"postFilterChunks", preFilterChunks,
+			"filteredChunks", 0,
+			"preFilterSeries", preFilterSeries,
+			"postFilterSeries", preFilterSeries,
+			"filteredSeries", 0,
+		)
+
+		bq.metrics.chunksTotal.Add(float64(preFilterChunks))
+		bq.metrics.chunksFiltered.Add(0)
+		bq.metrics.seriesTotal.Add(float64(preFilterSeries))
+		bq.metrics.seriesFiltered.Add(0)
+
+		return chunkRefs, false, nil
 	}
 
 	var skippedGrps [][]*logproto.GroupedChunkRefs
diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index 6ad0adde14cb6..bd2854f4752c6 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -1542,7 +1542,8 @@ func (t *Loki) initIndexGateway() (services.Service, error) {
 		}
 		resolver := bloomgateway.NewBlockResolver(t.BloomStore, logger)
 		querierCfg := bloomgateway.QuerierConfig{
-			MinTableOffset: t.Cfg.BloomBuild.Planner.MinTableOffset,
+			BuildTableOffset: t.Cfg.BloomBuild.Planner.MinTableOffset,
+			BuildInterval:    t.Cfg.BloomBuild.Planner.PlanningInterval,
 		}
 		bloomQuerier = bloomgateway.NewQuerier(bloomGatewayClient, querierCfg, t.Overrides, resolver, prometheus.DefaultRegisterer, logger)
 	}

From 457c662d4825f7b793b4d70097bac3dc03a5957c Mon Sep 17 00:00:00 2001
From: Vojtech Splichal <splichal@gmail.com>
Date: Fri, 6 Dec 2024 20:04:38 +0100
Subject: [PATCH 07/14] docs: correct dates in 2.9.x release notes (#15297)

Co-authored-by: J Stickler <julie.stickler@grafana.com>
---
 docs/sources/release-notes/v2-9.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/sources/release-notes/v2-9.md b/docs/sources/release-notes/v2-9.md
index 6be5e268e576a..51d9ee71bdb15 100644
--- a/docs/sources/release-notes/v2-9.md
+++ b/docs/sources/release-notes/v2-9.md
@@ -36,14 +36,14 @@ For a full list of all changes and fixes, refer to the [CHANGELOG](https://githu
 
 ## Bug fixes
 
-### 2.9.11 (2026-12-05)
+### 2.9.11 (2024-12-04)
 
 - **docker:** Update Docker to 23.0.15 ([#](https://github.com/grafana/loki/issues/)).
 - **lamba-promtail:** Lamba-promtail updates, some of which address CVEs([#14105](https://github.com/grafana/loki/issues/14105)).
 - **promtail:** Switch Promtail base image from Debian to Ubuntu to fix critical security issues ([#15195](https://github.com/grafana/loki/issues/15195)).
 - **storage:** Fix bug in cache of the index object client([#10585](https://github.com/grafana/loki/issues/10585)).
 
-### 2.9.10 (2026-08-09)
+### 2.9.10 (2024-08-09)
 
 - Update dependencies versions to remove CVE ([#13835](https://github.com/grafana/loki/pull/13835)) ([567bef2](https://github.com/grafana/loki/commit/567bef286376663407c54f5da07fa00963ba5485)).
 
@@ -104,4 +104,4 @@ For a full list of all changes and fixes, refer to the [CHANGELOG](https://githu
 - Updated Docker base images to mitigate security vulnerability CVE-2022-48174
 - Fixed bugs in indexshipper (`tsdb`, `boltdb-shipper`) that could result in not showing all ingested logs in query results.
 
-For a full list of all changes and fixes, refer to the [CHANGELOG](https://github.com/grafana/loki/blob/release-2.9.x/CHANGELOG.md).
\ No newline at end of file
+For a full list of all changes and fixes, refer to the [CHANGELOG](https://github.com/grafana/loki/blob/release-2.9.x/CHANGELOG.md).

From 17f197206d9e6f001754d854d26ba6c08b3c6934 Mon Sep 17 00:00:00 2001
From: Trevor Whitney <trevorjwhitney@gmail.com>
Date: Fri, 6 Dec 2024 16:30:00 -0700
Subject: [PATCH 08/14] fix: allow comparison against zero bytes (#15217)

Co-authored-by: George Robinson <george.robinson@grafana.com>
---
 pkg/logql/log/metrics_extraction_test.go | 16 ++++++++++++++++
 pkg/logql/syntax/lex_test.go             |  4 ++++
 pkg/logql/syntax/query_scanner.go        |  6 ++++--
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/pkg/logql/log/metrics_extraction_test.go b/pkg/logql/log/metrics_extraction_test.go
index 9cc5ff4411f51..00ebb3e9dee0c 100644
--- a/pkg/logql/log/metrics_extraction_test.go
+++ b/pkg/logql/log/metrics_extraction_test.go
@@ -162,6 +162,22 @@ func Test_labelSampleExtractor_Extract(t *testing.T) {
 			),
 			wantOk: true,
 		},
+		{
+			name: "convert bytes without spaces",
+			ex: mustSampleExtractor(LabelExtractorWithStages(
+				"foo", ConvertBytes, []string{"bar", "buzz"}, false, false, nil, NoopStage,
+			)),
+			in: labels.FromStrings("foo", "13MiB",
+				"bar", "foo",
+				"buzz", "blip",
+				"namespace", "dev",
+			),
+			want: 13 * 1024 * 1024,
+			wantLbs: labels.FromStrings("bar", "foo",
+				"buzz", "blip",
+			),
+			wantOk: true,
+		},
 		{
 			name: "convert bytes with structured metadata",
 			ex: mustSampleExtractor(LabelExtractorWithStages(
diff --git a/pkg/logql/syntax/lex_test.go b/pkg/logql/syntax/lex_test.go
index 08878186bd89b..0e9ad2aac3338 100644
--- a/pkg/logql/syntax/lex_test.go
+++ b/pkg/logql/syntax/lex_test.go
@@ -108,6 +108,10 @@ func TestLex(t *testing.T) {
 		{`34+-123`, []int{NUMBER, ADD, SUB, NUMBER}},
 		{`34-123`, []int{NUMBER, SUB, NUMBER}},
 		{`sum(rate({foo="bar"}[5m])-1 > 30`, []int{SUM, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, RANGE, CLOSE_PARENTHESIS, SUB, NUMBER, GT, NUMBER}},
+		{`{foo="bar"} | logfmt | bytes  < 0B`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, LT, BYTES}},
+		{`{foo="bar"} | logfmt | bytes  < 1B`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, LT, BYTES}},
+		{`0b01`, []int{NUMBER}},
+		{`0b10`, []int{NUMBER}},
 	} {
 		t.Run(tc.input, func(t *testing.T) {
 			actual := []int{}
diff --git a/pkg/logql/syntax/query_scanner.go b/pkg/logql/syntax/query_scanner.go
index 8d48e2e33d48d..23f79104591b7 100644
--- a/pkg/logql/syntax/query_scanner.go
+++ b/pkg/logql/syntax/query_scanner.go
@@ -409,8 +409,10 @@ func (s *Scanner) scanNumber(ch rune, seenDot bool) (rune, rune) {
 				ch = s.next()
 				base, prefix = 8, 'o'
 			case 'b':
-				ch = s.next()
-				base, prefix = 2, 'b'
+				if peek := s.Peek(); peek == '0' || peek == '1' {
+					ch = s.next()
+					base, prefix = 2, 'b'
+				}
 			default:
 				base, prefix = 8, '0'
 				digsep = 1 // leading 0

From c519ab6f18f64f6f9180bd1a96eb70f6eed3c8df Mon Sep 17 00:00:00 2001
From: Ashwanth <iamashwanth@gmail.com>
Date: Mon, 9 Dec 2024 08:17:18 +0530
Subject: [PATCH 09/14] feat(scheduler): implement and register block builder
 rpc service (#15248)

---
 docs/sources/shared/configuration.md     |  4 +++
 pkg/blockbuilder/builder/builder.go      | 32 ++++++++++++-----
 pkg/blockbuilder/builder/builder_test.go | 16 ---------
 pkg/blockbuilder/scheduler/scheduler.go  | 45 ++++++++++++++++++------
 pkg/blockbuilder/scheduler/strategy.go   |  2 +-
 pkg/blockbuilder/types/grpc_transport.go |  4 +--
 pkg/blockbuilder/types/job.go            |  6 ++--
 pkg/loki/modules.go                      | 11 +++++-
 8 files changed, 78 insertions(+), 42 deletions(-)
 delete mode 100644 pkg/blockbuilder/builder/builder_test.go

diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md
index 1160adff8f8ba..47587bce9fa81 100644
--- a/docs/sources/shared/configuration.md
+++ b/docs/sources/shared/configuration.md
@@ -192,6 +192,10 @@ block_builder:
   # CLI flag: -blockbuilder.sync-interval
   [sync_interval: <duration> | default = 30s]
 
+  # The interval at which to poll for new jobs.
+  # CLI flag: -blockbuilder.poll-interval
+  [poll_interval: <duration> | default = 30s]
+
   # Address of the scheduler in the format described here:
   # https://github.com/grpc/grpc/blob/master/doc/naming.md
   # CLI flag: -blockbuilder.scheduler-address
diff --git a/pkg/blockbuilder/builder/builder.go b/pkg/blockbuilder/builder/builder.go
index e3c1420b3cd81..aaeaad0d864fb 100644
--- a/pkg/blockbuilder/builder/builder.go
+++ b/pkg/blockbuilder/builder/builder.go
@@ -42,6 +42,7 @@ type Config struct {
 	Backoff           backoff.Config `yaml:"backoff_config"`
 	WorkerParallelism int            `yaml:"worker_parallelism"`
 	SyncInterval      time.Duration  `yaml:"sync_interval"`
+	PollInterval      time.Duration  `yaml:"poll_interval"`
 
 	SchedulerAddress string `yaml:"scheduler_address"`
 	// SchedulerGRPCClientConfig configures the gRPC connection between the block-builder and its scheduler.
@@ -58,6 +59,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.ChunkEncoding, prefix+"chunk-encoding", compression.Snappy.String(), fmt.Sprintf("The algorithm to use for compressing chunk. (%s)", compression.SupportedCodecs()))
 	f.DurationVar(&cfg.MaxChunkAge, prefix+"max-chunk-age", 2*time.Hour, "The maximum duration of a timeseries chunk in memory. If a timeseries runs for longer than this, the current chunk will be flushed to the store and a new chunk created.")
 	f.DurationVar(&cfg.SyncInterval, prefix+"sync-interval", 30*time.Second, "The interval at which to sync job status with the scheduler.")
+	f.DurationVar(&cfg.PollInterval, prefix+"poll-interval", 30*time.Second, "The interval at which to poll for new jobs.")
 	f.IntVar(&cfg.WorkerParallelism, prefix+"worker-parallelism", 1, "The number of workers to run in parallel to process jobs.")
 	f.StringVar(&cfg.SchedulerAddress, prefix+"scheduler-address", "", "Address of the scheduler in the format described here: https://github.com/grpc/grpc/blob/master/doc/naming.md")
 
@@ -81,6 +83,10 @@ func (cfg *Config) Validate() error {
 		return errors.New("sync interval must be greater than 0")
 	}
 
+	if cfg.PollInterval <= 0 {
+		return errors.New("poll interval must be greater than 0")
+	}
+
 	if cfg.WorkerParallelism < 1 {
 		return errors.New("worker parallelism must be greater than 0")
 	}
@@ -165,15 +171,23 @@ func (i *BlockBuilder) running(ctx context.Context) error {
 		go func(id string) {
 			defer wg.Done()
 
+			var waitFor time.Duration
 			for {
 				select {
 				case <-ctx.Done():
 					return
-				default:
-					err := i.runOne(ctx, id)
+				case <-time.After(waitFor):
+					gotJob, err := i.runOne(ctx, id)
 					if err != nil {
 						level.Error(i.logger).Log("msg", "block builder run failed", "err", err)
 					}
+
+					// poll only when there are no jobs
+					if gotJob {
+						waitFor = 0
+					} else {
+						waitFor = i.cfg.PollInterval
+					}
 				}
 			}
 		}(fmt.Sprintf("worker-%d", j))
@@ -218,18 +232,18 @@ func (i *BlockBuilder) syncJobs(ctx context.Context) error {
 	return nil
 }
 
-func (i *BlockBuilder) runOne(ctx context.Context, workerID string) error {
+func (i *BlockBuilder) runOne(ctx context.Context, workerID string) (bool, error) {
 	// assuming GetJob blocks/polls until a job is available
 	resp, err := i.SendGetJobRequest(ctx, &types.GetJobRequest{
 		BuilderID: workerID,
 	})
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	if !resp.OK {
 		level.Info(i.logger).Log("msg", "no available job to process")
-		return nil
+		return false, nil
 	}
 
 	job := resp.Job
@@ -262,7 +276,7 @@ func (i *BlockBuilder) runOne(ctx context.Context, workerID string) error {
 			return
 		},
 	); err != nil {
-		return err
+		return true, err
 	}
 
 	i.jobsMtx.Lock()
@@ -270,7 +284,7 @@ func (i *BlockBuilder) runOne(ctx context.Context, workerID string) error {
 	i.metrics.inflightJobs.Set(float64(len(i.inflightJobs)))
 	i.jobsMtx.Unlock()
 
-	return err
+	return true, err
 }
 
 func (i *BlockBuilder) processJob(ctx context.Context, job *types.Job, logger log.Logger) (lastOffsetConsumed int64, err error) {
@@ -297,7 +311,7 @@ func (i *BlockBuilder) processJob(ctx context.Context, job *types.Job, logger lo
 		"load records",
 		1,
 		func(ctx context.Context) error {
-			lastOffset, err = i.loadRecords(ctx, int32(job.Partition), job.Offsets, inputCh)
+			lastOffset, err = i.loadRecords(ctx, job.Partition, job.Offsets, inputCh)
 			return err
 		},
 		func(ctx context.Context) error {
@@ -545,7 +559,7 @@ func (i *BlockBuilder) loadRecords(ctx context.Context, partitionID int32, offse
 		}
 	}
 
-	return lastOffset, err
+	return lastOffset, boff.Err()
 }
 
 func withBackoff[T any](
diff --git a/pkg/blockbuilder/builder/builder_test.go b/pkg/blockbuilder/builder/builder_test.go
deleted file mode 100644
index ac9890526f1d7..0000000000000
--- a/pkg/blockbuilder/builder/builder_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package builder
-
-import (
-	"github.com/grafana/loki/v3/pkg/blockbuilder/types"
-)
-
-// TestBuilder implements Worker interface for testing
-type TestBuilder struct {
-	*Worker
-}
-
-func NewTestBuilder(builderID string, transport types.Transport) *TestBuilder {
-	return &TestBuilder{
-		Worker: NewWorker(builderID, transport),
-	}
-}
diff --git a/pkg/blockbuilder/scheduler/scheduler.go b/pkg/blockbuilder/scheduler/scheduler.go
index 632e6842993a5..e2272a039f5c4 100644
--- a/pkg/blockbuilder/scheduler/scheduler.go
+++ b/pkg/blockbuilder/scheduler/scheduler.go
@@ -16,10 +16,10 @@ import (
 	"github.com/twmb/franz-go/pkg/kadm"
 
 	"github.com/grafana/loki/v3/pkg/blockbuilder/types"
+	"github.com/grafana/loki/v3/pkg/blockbuilder/types/proto"
 )
 
 var (
-	_ types.Scheduler = unimplementedScheduler{}
 	_ types.Scheduler = &BlockScheduler{}
 )
 
@@ -185,17 +185,42 @@ func (s *BlockScheduler) HandleSyncJob(_ context.Context, builderID string, job
 	return nil
 }
 
-// unimplementedScheduler provides default implementations that panic.
-type unimplementedScheduler struct{}
-
-func (s unimplementedScheduler) HandleGetJob(_ context.Context, _ string) (*types.Job, bool, error) {
-	panic("unimplemented")
+func (s *BlockScheduler) CompleteJob(_ context.Context, req *proto.CompleteJobRequest) (*proto.CompleteJobResponse, error) {
+	s.queue.MarkComplete(req.Job.Id)
+	return &proto.CompleteJobResponse{}, nil
 }
 
-func (s unimplementedScheduler) HandleCompleteJob(_ context.Context, _ string, _ *types.Job) error {
-	panic("unimplemented")
+func (s *BlockScheduler) SyncJob(_ context.Context, req *proto.SyncJobRequest) (*proto.SyncJobResponse, error) {
+	s.queue.SyncJob(req.Job.Id, req.BuilderId, &types.Job{
+		ID:        req.Job.Id,
+		Partition: req.Job.Partition,
+		Offsets: types.Offsets{
+			Min: req.Job.Offsets.Min,
+			Max: req.Job.Offsets.Max,
+		},
+	})
+
+	return &proto.SyncJobResponse{}, nil
 }
 
-func (s unimplementedScheduler) HandleSyncJob(_ context.Context, _ string, _ *types.Job) error {
-	panic("unimplemented")
+func (s *BlockScheduler) GetJob(_ context.Context, req *proto.GetJobRequest) (*proto.GetJobResponse, error) {
+	var resp proto.GetJobResponse
+	job, ok, err := s.queue.Dequeue(req.BuilderId)
+	if err != nil {
+		return &resp, err
+	}
+
+	if ok {
+		resp.Ok = true
+		resp.Job = &proto.Job{
+			Id:        job.ID,
+			Partition: job.Partition,
+			Offsets: &proto.Offsets{
+				Min: job.Offsets.Min,
+				Max: job.Offsets.Max,
+			},
+		}
+	}
+
+	return &resp, nil
 }
diff --git a/pkg/blockbuilder/scheduler/strategy.go b/pkg/blockbuilder/scheduler/strategy.go
index 0b8c38aa64d11..75719140a4ea0 100644
--- a/pkg/blockbuilder/scheduler/strategy.go
+++ b/pkg/blockbuilder/scheduler/strategy.go
@@ -70,7 +70,7 @@ func (p *RecordCountPlanner) Plan(ctx context.Context) ([]*JobWithPriority[int],
 			currentEnd := min(currentStart+p.targetRecordCount, endOffset)
 
 			job := NewJobWithPriority(
-				types.NewJob(int(partitionOffset.Partition), types.Offsets{
+				types.NewJob(partitionOffset.Partition, types.Offsets{
 					Min: currentStart,
 					Max: currentEnd,
 				}), int(endOffset-currentStart), // priority is remaining records to process
diff --git a/pkg/blockbuilder/types/grpc_transport.go b/pkg/blockbuilder/types/grpc_transport.go
index a5a9fa843671e..b083cd76bb37a 100644
--- a/pkg/blockbuilder/types/grpc_transport.go
+++ b/pkg/blockbuilder/types/grpc_transport.go
@@ -111,7 +111,7 @@ func protoToJob(p *proto.Job) *Job {
 	}
 	return &Job{
 		ID:        p.GetId(),
-		Partition: int(p.GetPartition()),
+		Partition: p.GetPartition(),
 		Offsets: Offsets{
 			Min: p.GetOffsets().GetMin(),
 			Max: p.GetOffsets().GetMax(),
@@ -126,7 +126,7 @@ func jobToProto(j *Job) *proto.Job {
 	}
 	return &proto.Job{
 		Id:        j.ID,
-		Partition: int32(j.Partition),
+		Partition: j.Partition,
 		Offsets: &proto.Offsets{
 			Min: j.Offsets.Min,
 			Max: j.Offsets.Max,
diff --git a/pkg/blockbuilder/types/job.go b/pkg/blockbuilder/types/job.go
index 2c06fec4d48cd..9cf94daebd484 100644
--- a/pkg/blockbuilder/types/job.go
+++ b/pkg/blockbuilder/types/job.go
@@ -6,7 +6,7 @@ import "fmt"
 type Job struct {
 	ID string
 	// Partition and offset information
-	Partition int
+	Partition int32
 	Offsets   Offsets
 }
 
@@ -26,7 +26,7 @@ type Offsets struct {
 }
 
 // NewJob creates a new job with the given partition and offsets
-func NewJob(partition int, offsets Offsets) *Job {
+func NewJob(partition int32, offsets Offsets) *Job {
 	return &Job{
 		ID:        GenerateJobID(partition, offsets),
 		Partition: partition,
@@ -35,6 +35,6 @@ func NewJob(partition int, offsets Offsets) *Job {
 }
 
 // GenerateJobID creates a deterministic job ID from partition and offsets
-func GenerateJobID(partition int, offsets Offsets) string {
+func GenerateJobID(partition int32, offsets Offsets) string {
 	return fmt.Sprintf("job-%d-%d-%d", partition, offsets.Min, offsets.Max)
 }
diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index bd2854f4752c6..9b521eb7fc083 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -38,6 +38,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/analytics"
 	blockbuilder "github.com/grafana/loki/v3/pkg/blockbuilder/builder"
 	blockscheduler "github.com/grafana/loki/v3/pkg/blockbuilder/scheduler"
+	blockprotos "github.com/grafana/loki/v3/pkg/blockbuilder/types/proto"
 	"github.com/grafana/loki/v3/pkg/bloombuild/builder"
 	"github.com/grafana/loki/v3/pkg/bloombuild/planner"
 	bloomprotos "github.com/grafana/loki/v3/pkg/bloombuild/protos"
@@ -1862,8 +1863,16 @@ func (t *Loki) initBlockScheduler() (services.Service, error) {
 	if err != nil {
 		return nil, fmt.Errorf("creating kafka client: %w", err)
 	}
+
 	offsetReader := blockscheduler.NewOffsetReader(t.Cfg.KafkaConfig.Topic, t.Cfg.BlockScheduler.ConsumerGroup, t.Cfg.BlockScheduler.LookbackPeriod, c)
-	return blockscheduler.NewScheduler(t.Cfg.BlockScheduler, blockscheduler.NewJobQueue(), offsetReader, logger, prometheus.DefaultRegisterer), nil
+	s, err := blockscheduler.NewScheduler(t.Cfg.BlockScheduler, blockscheduler.NewJobQueue(), offsetReader, logger, prometheus.DefaultRegisterer), nil
+	if err != nil {
+		return s, err
+	}
+
+	blockprotos.RegisterBlockBuilderServiceServer(t.Server.GRPC, s)
+
+	return s, err
 }
 
 func (t *Loki) deleteRequestsClient(clientType string, limits limiter.CombinedLimits) (deletion.DeleteRequestsClient, error) {

From 0ddaca889cf8b3c00375183ebbfd952ec12e1bad Mon Sep 17 00:00:00 2001
From: Owen Diehl <ow.diehl@gmail.com>
Date: Mon, 9 Dec 2024 12:46:21 +0800
Subject: [PATCH 10/14] refactor(blockbuilder): transport job completion
 reports success (#15313)

---
 pkg/blockbuilder/builder/builder.go           | 16 +--
 pkg/blockbuilder/builder/worker.go            |  5 +-
 pkg/blockbuilder/scheduler/scheduler_test.go  |  6 +-
 pkg/blockbuilder/types/grpc_transport.go      |  6 +-
 pkg/blockbuilder/types/interfaces.go          |  8 +-
 .../types/proto/blockbuilder.pb.go            | 98 ++++++++++---------
 .../types/proto/blockbuilder.proto            |  2 +-
 7 files changed, 76 insertions(+), 65 deletions(-)

diff --git a/pkg/blockbuilder/builder/builder.go b/pkg/blockbuilder/builder/builder.go
index aaeaad0d864fb..d8784bf568d99 100644
--- a/pkg/blockbuilder/builder/builder.go
+++ b/pkg/blockbuilder/builder/builder.go
@@ -260,17 +260,21 @@ func (i *BlockBuilder) runOne(ctx context.Context, workerID string) (bool, error
 	i.metrics.inflightJobs.Set(float64(len(i.inflightJobs)))
 	i.jobsMtx.Unlock()
 
-	lastConsumedOffset, err := i.processJob(ctx, job, logger)
+	completion := &types.CompleteJobRequest{
+		BuilderID: workerID,
+		Job:       job,
+		Success:   true,
+	}
+	if _, err = i.processJob(ctx, job, logger); err != nil {
+		level.Error(i.logger).Log("msg", "failed to process job", "err", err)
+		completion.Success = false
+	}
 
 	if _, err := withBackoff(
 		ctx,
 		i.cfg.Backoff,
 		func() (res struct{}, err error) {
-			if err = i.SendCompleteJob(ctx, &types.CompleteJobRequest{
-				BuilderID:          workerID,
-				Job:                job,
-				LastConsumedOffset: lastConsumedOffset,
-			}); err != nil {
+			if err = i.SendCompleteJob(ctx, completion); err != nil {
 				level.Error(i.logger).Log("msg", "failed to mark the job as complete", "err", err)
 			}
 			return
diff --git a/pkg/blockbuilder/builder/worker.go b/pkg/blockbuilder/builder/worker.go
index 41dc0cb7563ec..4b5f36d6492b9 100644
--- a/pkg/blockbuilder/builder/worker.go
+++ b/pkg/blockbuilder/builder/worker.go
@@ -18,7 +18,7 @@ func (u unimplementedWorker) GetJob(_ context.Context) (*types.Job, bool, error)
 	panic("unimplemented")
 }
 
-func (u unimplementedWorker) CompleteJob(_ context.Context, _ *types.Job) error {
+func (u unimplementedWorker) CompleteJob(_ context.Context, _ *types.Job, _ bool) error {
 	panic("unimplemented")
 }
 
@@ -51,10 +51,11 @@ func (w *Worker) GetJob(ctx context.Context) (*types.Job, bool, error) {
 	return resp.Job, resp.OK, nil
 }
 
-func (w *Worker) CompleteJob(ctx context.Context, job *types.Job) error {
+func (w *Worker) CompleteJob(ctx context.Context, job *types.Job, success bool) error {
 	return w.transport.SendCompleteJob(ctx, &types.CompleteJobRequest{
 		BuilderID: w.builderID,
 		Job:       job,
+		Success:   success,
 	})
 }
 
diff --git a/pkg/blockbuilder/scheduler/scheduler_test.go b/pkg/blockbuilder/scheduler/scheduler_test.go
index 977dd1d556d1e..201b56eeb9786 100644
--- a/pkg/blockbuilder/scheduler/scheduler_test.go
+++ b/pkg/blockbuilder/scheduler/scheduler_test.go
@@ -57,7 +57,7 @@ func TestScheduleAndProcessJob(t *testing.T) {
 	}
 
 	// Builder completes job
-	err = env.builder.CompleteJob(ctx, receivedJob)
+	err = env.builder.CompleteJob(ctx, receivedJob, true)
 	if err != nil {
 		t.Fatalf("failed to complete job: %v", err)
 	}
@@ -130,12 +130,12 @@ func TestMultipleBuilders(t *testing.T) {
 	}
 
 	// Complete jobs
-	err = env1.builder.CompleteJob(ctx, receivedJob1)
+	err = env1.builder.CompleteJob(ctx, receivedJob1, true)
 	if err != nil {
 		t.Fatalf("builder1 failed to complete job: %v", err)
 	}
 
-	err = builder2.CompleteJob(ctx, receivedJob2)
+	err = builder2.CompleteJob(ctx, receivedJob2, true)
 	if err != nil {
 		t.Fatalf("builder2 failed to complete job: %v", err)
 	}
diff --git a/pkg/blockbuilder/types/grpc_transport.go b/pkg/blockbuilder/types/grpc_transport.go
index b083cd76bb37a..a53f06282e895 100644
--- a/pkg/blockbuilder/types/grpc_transport.go
+++ b/pkg/blockbuilder/types/grpc_transport.go
@@ -84,9 +84,9 @@ func (t *GRPCTransport) SendGetJobRequest(ctx context.Context, req *GetJobReques
 // SendCompleteJob implements Transport
 func (t *GRPCTransport) SendCompleteJob(ctx context.Context, req *CompleteJobRequest) error {
 	protoReq := &proto.CompleteJobRequest{
-		BuilderId:          req.BuilderID,
-		Job:                jobToProto(req.Job),
-		LastConsumedOffset: req.LastConsumedOffset,
+		BuilderId: req.BuilderID,
+		Job:       jobToProto(req.Job),
+		Success:   req.Success,
 	}
 
 	_, err := t.CompleteJob(ctx, protoReq)
diff --git a/pkg/blockbuilder/types/interfaces.go b/pkg/blockbuilder/types/interfaces.go
index e3e4a8476e3ca..f5746dea5fbc2 100644
--- a/pkg/blockbuilder/types/interfaces.go
+++ b/pkg/blockbuilder/types/interfaces.go
@@ -7,7 +7,7 @@ type Worker interface {
 	// GetJob requests a new job from the scheduler
 	GetJob(ctx context.Context) (*Job, bool, error)
 	// CompleteJob marks a job as finished
-	CompleteJob(ctx context.Context, job *Job) error
+	CompleteJob(ctx context.Context, job *Job, success bool) error
 	// SyncJob informs the scheduler about an in-progress job
 	SyncJob(ctx context.Context, job *Job) error
 }
@@ -52,9 +52,9 @@ type GetJobResponse struct {
 }
 
 type CompleteJobRequest struct {
-	BuilderID          string
-	Job                *Job
-	LastConsumedOffset int64
+	BuilderID string
+	Job       *Job
+	Success   bool
 }
 
 type SyncJobRequest struct {
diff --git a/pkg/blockbuilder/types/proto/blockbuilder.pb.go b/pkg/blockbuilder/types/proto/blockbuilder.pb.go
index b6a743a4b571f..071812c2fa189 100644
--- a/pkg/blockbuilder/types/proto/blockbuilder.pb.go
+++ b/pkg/blockbuilder/types/proto/blockbuilder.pb.go
@@ -126,9 +126,9 @@ func (m *GetJobResponse) GetOk() bool {
 
 // CompleteJobRequest represents a job completion notification
 type CompleteJobRequest struct {
-	BuilderId          string `protobuf:"bytes,1,opt,name=builder_id,json=builderId,proto3" json:"builder_id,omitempty"`
-	Job                *Job   `protobuf:"bytes,2,opt,name=job,proto3" json:"job,omitempty"`
-	LastConsumedOffset int64  `protobuf:"varint,3,opt,name=LastConsumedOffset,proto3" json:"LastConsumedOffset,omitempty"`
+	BuilderId string `protobuf:"bytes,1,opt,name=builder_id,json=builderId,proto3" json:"builder_id,omitempty"`
+	Job       *Job   `protobuf:"bytes,2,opt,name=job,proto3" json:"job,omitempty"`
+	Success   bool   `protobuf:"varint,3,opt,name=success,proto3" json:"success,omitempty"`
 }
 
 func (m *CompleteJobRequest) Reset()      { *m = CompleteJobRequest{} }
@@ -177,11 +177,11 @@ func (m *CompleteJobRequest) GetJob() *Job {
 	return nil
 }
 
-func (m *CompleteJobRequest) GetLastConsumedOffset() int64 {
+func (m *CompleteJobRequest) GetSuccess() bool {
 	if m != nil {
-		return m.LastConsumedOffset
+		return m.Success
 	}
-	return 0
+	return false
 }
 
 // CompleteJobResponse is an empty response for job completion
@@ -436,36 +436,36 @@ func init() {
 }
 
 var fileDescriptor_04968622516f7b79 = []byte{
-	// 464 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x93, 0x31, 0x8e, 0xd3, 0x40,
-	0x14, 0x86, 0x3d, 0xb6, 0xd8, 0x90, 0x17, 0x11, 0x60, 0x56, 0x88, 0x28, 0xc0, 0x28, 0x0c, 0x12,
-	0x84, 0x02, 0x5b, 0x0a, 0x70, 0x81, 0x6c, 0x81, 0x58, 0x90, 0x10, 0x5e, 0xaa, 0x6d, 0xc0, 0x8e,
-	0x27, 0x61, 0xe2, 0xc4, 0x63, 0x3c, 0x13, 0xb4, 0xdb, 0x71, 0x02, 0xc4, 0x11, 0x28, 0x39, 0x0a,
-	0x65, 0xca, 0x2d, 0x89, 0xd3, 0x50, 0xee, 0x11, 0x90, 0xc7, 0x76, 0xc0, 0x5a, 0x2b, 0xa4, 0xd9,
-	0xca, 0xd6, 0x3f, 0x9f, 0xdf, 0xff, 0xfb, 0xbd, 0x37, 0xe0, 0xc4, 0xe1, 0xc4, 0xf1, 0x67, 0x62,
-	0x14, 0xfa, 0x0b, 0x3e, 0x0b, 0x58, 0xe2, 0xa8, 0xd3, 0x98, 0x49, 0x27, 0x4e, 0x84, 0x12, 0x95,
-	0x03, 0x5b, 0x4b, 0x18, 0x57, 0x34, 0x0d, 0x53, 0x1b, 0xae, 0xbd, 0x60, 0xea, 0x50, 0xf8, 0x2e,
-	0xfb, 0xb4, 0x60, 0x52, 0xe1, 0x7b, 0x00, 0x05, 0xf1, 0x9e, 0x07, 0x1d, 0xd4, 0x43, 0xfd, 0xa6,
-	0xdb, 0x2c, 0x94, 0x97, 0x01, 0x7d, 0x05, 0xed, 0x92, 0x97, 0xb1, 0x88, 0x24, 0xc3, 0x8f, 0xc1,
-	0x9a, 0x0a, 0x5f, 0x93, 0xad, 0xc1, 0x6d, 0xfb, 0xa2, 0x87, 0x9d, 0xd1, 0x19, 0x83, 0xdb, 0x60,
-	0x8a, 0xb0, 0x63, 0xf6, 0x50, 0xff, 0xaa, 0x6b, 0x8a, 0x90, 0x7e, 0x45, 0x80, 0x0f, 0xc4, 0x3c,
-	0x9e, 0x31, 0xc5, 0x76, 0x8e, 0x50, 0x1a, 0x9a, 0x3b, 0x18, 0xda, 0x80, 0x5f, 0x7b, 0x52, 0x1d,
-	0x88, 0x48, 0x2e, 0xe6, 0x2c, 0x78, 0x33, 0x1e, 0x4b, 0xa6, 0x3a, 0x56, 0x0f, 0xf5, 0x2d, 0xb7,
-	0xe6, 0x84, 0xde, 0x82, 0xfd, 0x4a, 0x9e, 0xfc, 0x17, 0xe9, 0x31, 0xb4, 0x8f, 0x4e, 0xa3, 0xd1,
-	0x65, 0x44, 0xa4, 0x37, 0xe1, 0xfa, 0xa6, 0x76, 0x61, 0xf7, 0x04, 0x1a, 0x79, 0x1e, 0x89, 0x6f,
-	0x80, 0x35, 0xe7, 0x91, 0x36, 0xb0, 0xdc, 0xec, 0x55, 0x2b, 0xde, 0x89, 0x2e, 0x9d, 0x29, 0xde,
-	0x09, 0x9d, 0x82, 0x75, 0x98, 0x37, 0x77, 0x13, 0xc5, 0xe4, 0x01, 0xbe, 0x0b, 0xcd, 0xd8, 0x4b,
-	0x14, 0x57, 0x5c, 0x44, 0x1a, 0xbf, 0xe2, 0xfe, 0x15, 0xf0, 0x73, 0x68, 0x88, 0xdc, 0x43, 0xb7,
-	0xa3, 0x35, 0xb8, 0x53, 0x97, 0xb2, 0x88, 0xe1, 0x96, 0xec, 0xe0, 0xbb, 0x09, 0xfb, 0xc3, 0x8c,
-	0x1b, 0xe6, 0xdc, 0x11, 0x4b, 0x3e, 0xf3, 0x11, 0xc3, 0x6f, 0x61, 0x2f, 0x5f, 0x0b, 0x7c, 0xbf,
-	0xae, 0x4e, 0x65, 0xc5, 0xba, 0x74, 0x1b, 0x52, 0xf4, 0xc0, 0xc0, 0x1f, 0xa0, 0xf5, 0xcf, 0x2c,
-	0xf0, 0xc3, 0xba, 0x8f, 0x2e, 0x2e, 0x4f, 0xf7, 0xd1, 0x7f, 0xb9, 0x8d, 0xc3, 0x3b, 0x68, 0x14,
-	0xad, 0xc7, 0xb5, 0x91, 0xaa, 0x33, 0xef, 0x3e, 0xd8, 0xca, 0x94, 0x55, 0x87, 0xd3, 0xe5, 0x8a,
-	0x18, 0x67, 0x2b, 0x62, 0x9c, 0xaf, 0x08, 0xfa, 0x92, 0x12, 0xf4, 0x23, 0x25, 0xe8, 0x67, 0x4a,
-	0xd0, 0x32, 0x25, 0xe8, 0x57, 0x4a, 0xd0, 0xef, 0x94, 0x18, 0xe7, 0x29, 0x41, 0xdf, 0xd6, 0xc4,
-	0x58, 0xae, 0x89, 0x71, 0xb6, 0x26, 0xc6, 0xf1, 0xb3, 0x09, 0x57, 0x1f, 0x17, 0xbe, 0x3d, 0x12,
-	0x73, 0x67, 0x92, 0x78, 0x63, 0x2f, 0xf2, 0x9c, 0x99, 0x08, 0xf9, 0xd6, 0x4b, 0xee, 0xef, 0xe9,
-	0xc7, 0xd3, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x20, 0x0d, 0x30, 0x7c, 0x0b, 0x04, 0x00, 0x00,
+	// 453 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x93, 0xc1, 0x8e, 0x93, 0x40,
+	0x18, 0xc7, 0x19, 0x88, 0x5b, 0xfb, 0x35, 0x56, 0x9d, 0x8d, 0xb1, 0xa9, 0x3a, 0x59, 0xc7, 0x44,
+	0xd7, 0x83, 0x90, 0x54, 0x7d, 0x81, 0x7a, 0x30, 0xae, 0x07, 0x23, 0xeb, 0x69, 0x2f, 0x0a, 0x74,
+	0x5a, 0xa7, 0x50, 0x06, 0x99, 0xc1, 0x74, 0x6f, 0x3e, 0x82, 0x8f, 0xe0, 0xd1, 0x47, 0xf1, 0xd8,
+	0xe3, 0x1e, 0x2d, 0xbd, 0x78, 0xdc, 0x47, 0x30, 0x0c, 0x50, 0x25, 0x4b, 0xea, 0x5e, 0x3c, 0x01,
+	0x7f, 0x7e, 0x7c, 0xff, 0x3f, 0xdf, 0xf7, 0x0d, 0x38, 0x49, 0x38, 0x73, 0xfc, 0x48, 0x04, 0xa1,
+	0x9f, 0xf1, 0x68, 0xc2, 0x52, 0x47, 0x9d, 0x26, 0x4c, 0x3a, 0x49, 0x2a, 0x94, 0x68, 0xbc, 0xb0,
+	0xb5, 0x84, 0x71, 0x43, 0xd3, 0x30, 0xb5, 0xe1, 0xda, 0x4b, 0xa6, 0x8e, 0x84, 0xef, 0xb2, 0x4f,
+	0x19, 0x93, 0x0a, 0xdf, 0x03, 0xa8, 0x88, 0xf7, 0x7c, 0x32, 0x40, 0x07, 0xe8, 0xb0, 0xeb, 0x76,
+	0x2b, 0xe5, 0xd5, 0x84, 0xbe, 0x86, 0x7e, 0xcd, 0xcb, 0x44, 0xc4, 0x92, 0xe1, 0xc7, 0x60, 0xcd,
+	0x85, 0xaf, 0xc9, 0xde, 0xe8, 0xb6, 0x7d, 0xd1, 0xc3, 0x2e, 0xe8, 0x82, 0xc1, 0x7d, 0x30, 0x45,
+	0x38, 0x30, 0x0f, 0xd0, 0xe1, 0x55, 0xd7, 0x14, 0x21, 0x5d, 0x02, 0x7e, 0x21, 0x16, 0x49, 0xc4,
+	0x14, 0xbb, 0x74, 0x82, 0xda, 0xcf, 0xbc, 0x84, 0xdf, 0x00, 0x3a, 0x32, 0x0b, 0x02, 0x26, 0xe5,
+	0xc0, 0xd2, 0xa6, 0xf5, 0x23, 0xbd, 0x05, 0xfb, 0x0d, 0xe7, 0xf2, 0x5f, 0xe8, 0x09, 0xf4, 0x8f,
+	0x4f, 0xe3, 0xe0, 0x7f, 0x84, 0xa1, 0x37, 0xe1, 0xfa, 0xb6, 0x76, 0x65, 0xf7, 0x04, 0x3a, 0x6f,
+	0xa6, 0x53, 0xc9, 0x94, 0xc4, 0x37, 0xc0, 0x5a, 0xf0, 0x58, 0x1b, 0x58, 0x6e, 0x71, 0xab, 0x15,
+	0x6f, 0xa9, 0x4b, 0x17, 0x8a, 0xb7, 0xa4, 0x73, 0xb0, 0x8e, 0xca, 0x2e, 0x6e, 0xa3, 0x98, 0x7c,
+	0x82, 0xef, 0x42, 0x37, 0xf1, 0x52, 0xc5, 0x15, 0x17, 0xb1, 0xc6, 0xaf, 0xb8, 0x7f, 0x04, 0xfc,
+	0x1c, 0x3a, 0xa2, 0xf4, 0xd0, 0x3d, 0xe8, 0x8d, 0xee, 0xb4, 0xa5, 0xac, 0x62, 0xb8, 0x35, 0x3b,
+	0xfa, 0x66, 0xc2, 0xfe, 0xb8, 0xe0, 0xc6, 0x25, 0x77, 0xcc, 0xd2, 0xcf, 0x3c, 0x60, 0xf8, 0x2d,
+	0xec, 0x95, 0xf3, 0xc7, 0xf7, 0xdb, 0xea, 0x34, 0x76, 0x69, 0x48, 0x77, 0x21, 0x55, 0x0f, 0x0c,
+	0xfc, 0x01, 0x7a, 0x7f, 0xcd, 0x02, 0x3f, 0x6c, 0xfb, 0xe8, 0xe2, 0x9a, 0x0c, 0x1f, 0xfd, 0x93,
+	0xdb, 0x3a, 0xbc, 0x83, 0x4e, 0xd5, 0x7a, 0xdc, 0x1a, 0xa9, 0x39, 0xf3, 0xe1, 0x83, 0x9d, 0x4c,
+	0x5d, 0x75, 0x3c, 0x5f, 0xad, 0x89, 0x71, 0xb6, 0x26, 0xc6, 0xf9, 0x9a, 0xa0, 0x2f, 0x39, 0x41,
+	0xdf, 0x73, 0x82, 0x7e, 0xe4, 0x04, 0xad, 0x72, 0x82, 0x7e, 0xe6, 0x04, 0xfd, 0xca, 0x89, 0x71,
+	0x9e, 0x13, 0xf4, 0x75, 0x43, 0x8c, 0xd5, 0x86, 0x18, 0x67, 0x1b, 0x62, 0x9c, 0x3c, 0x9b, 0x71,
+	0xf5, 0x31, 0xf3, 0xed, 0x40, 0x2c, 0x9c, 0x59, 0xea, 0x4d, 0xbd, 0xd8, 0x73, 0x22, 0x11, 0xf2,
+	0x9d, 0xa7, 0xd9, 0xdf, 0xd3, 0x97, 0xa7, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xbf, 0xda, 0xc9,
+	0x32, 0xf4, 0x03, 0x00, 0x00,
 }
 
 func (this *GetJobRequest) Equal(that interface{}) bool {
@@ -544,7 +544,7 @@ func (this *CompleteJobRequest) Equal(that interface{}) bool {
 	if !this.Job.Equal(that1.Job) {
 		return false
 	}
-	if this.LastConsumedOffset != that1.LastConsumedOffset {
+	if this.Success != that1.Success {
 		return false
 	}
 	return true
@@ -708,7 +708,7 @@ func (this *CompleteJobRequest) GoString() string {
 	if this.Job != nil {
 		s = append(s, "Job: "+fmt.Sprintf("%#v", this.Job)+",\n")
 	}
-	s = append(s, "LastConsumedOffset: "+fmt.Sprintf("%#v", this.LastConsumedOffset)+",\n")
+	s = append(s, "Success: "+fmt.Sprintf("%#v", this.Success)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1030,8 +1030,13 @@ func (m *CompleteJobRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if m.LastConsumedOffset != 0 {
-		i = encodeVarintBlockbuilder(dAtA, i, uint64(m.LastConsumedOffset))
+	if m.Success {
+		i--
+		if m.Success {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
 		i--
 		dAtA[i] = 0x18
 	}
@@ -1279,8 +1284,8 @@ func (m *CompleteJobRequest) Size() (n int) {
 		l = m.Job.Size()
 		n += 1 + l + sovBlockbuilder(uint64(l))
 	}
-	if m.LastConsumedOffset != 0 {
-		n += 1 + sovBlockbuilder(uint64(m.LastConsumedOffset))
+	if m.Success {
+		n += 2
 	}
 	return n
 }
@@ -1389,7 +1394,7 @@ func (this *CompleteJobRequest) String() string {
 	s := strings.Join([]string{`&CompleteJobRequest{`,
 		`BuilderId:` + fmt.Sprintf("%v", this.BuilderId) + `,`,
 		`Job:` + strings.Replace(this.Job.String(), "Job", "Job", 1) + `,`,
-		`LastConsumedOffset:` + fmt.Sprintf("%v", this.LastConsumedOffset) + `,`,
+		`Success:` + fmt.Sprintf("%v", this.Success) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1747,9 +1752,9 @@ func (m *CompleteJobRequest) Unmarshal(dAtA []byte) error {
 			iNdEx = postIndex
 		case 3:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field LastConsumedOffset", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType)
 			}
-			m.LastConsumedOffset = 0
+			var v int
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
 					return ErrIntOverflowBlockbuilder
@@ -1759,11 +1764,12 @@ func (m *CompleteJobRequest) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.LastConsumedOffset |= int64(b&0x7F) << shift
+				v |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
 			}
+			m.Success = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipBlockbuilder(dAtA[iNdEx:])
diff --git a/pkg/blockbuilder/types/proto/blockbuilder.proto b/pkg/blockbuilder/types/proto/blockbuilder.proto
index 56160d336d336..a29b03dc6186f 100644
--- a/pkg/blockbuilder/types/proto/blockbuilder.proto
+++ b/pkg/blockbuilder/types/proto/blockbuilder.proto
@@ -29,7 +29,7 @@ message GetJobResponse {
 message CompleteJobRequest {
   string builder_id = 1;
   Job job = 2;
-  int64 LastConsumedOffset = 3;
+  bool success = 3;
 }
 
 // CompleteJobResponse is an empty response for job completion

From e784c934f05b3b363abc08cbc6d2fb7b89692fc5 Mon Sep 17 00:00:00 2001
From: Trevor Whitney <trevorjwhitney@gmail.com>
Date: Mon, 9 Dec 2024 09:49:40 -0700
Subject: [PATCH 11/14] ci: remove weekly builds of docker driver (#15317)

---
 .github/release-workflows.jsonnet |   6 -
 .github/workflows/images.yml      | 302 ------------------------------
 2 files changed, 308 deletions(-)

diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet
index bc00da6e2ad40..080dcae2d39c4 100644
--- a/.github/release-workflows.jsonnet
+++ b/.github/release-workflows.jsonnet
@@ -20,15 +20,9 @@ local imageJobs = {
 
 local weeklyImageJobs = {
   loki: build.weeklyImage('loki', 'cmd/loki'),
-  fluentd: build.weeklyImage('fluent-plugin-loki', 'clients/cmd/fluentd', platform=['linux/amd64']),
-  'fluent-bit': build.weeklyImage('fluent-bit-plugin-loki', 'clients/cmd/fluent-bit', platform=['linux/amd64']),
-  logstash: build.weeklyImage('logstash-output-loki', 'clients/cmd/logstash', platform=['linux/amd64']),
-  logcli: build.weeklyImage('logcli', 'cmd/logcli'),
   'loki-canary': build.weeklyImage('loki-canary', 'cmd/loki-canary'),
   'loki-canary-boringcrypto': build.weeklyImage('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'),
   promtail: build.weeklyImage('promtail', 'clients/cmd/promtail'),
-  querytee: build.weeklyImage('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']),
-  'loki-docker-driver': build.weeklyDockerPlugin('grafana/loki-docker-driver', 'clients/cmd/docker-driver', platform=['linux/amd64', 'linux/arm64']),
 };
 
 local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION');
diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml
index 64dac0158637a..55bbd64d6c3f9 100644
--- a/.github/workflows/images.yml
+++ b/.github/workflows/images.yml
@@ -7,190 +7,6 @@
       "release_lib_ref": "main"
       "skip_validation": false
       "use_github_app_token": true
-  "fluent-bit":
-    "env":
-      "BUILD_TIMEOUT": 60
-      "IMAGE_PREFIX": "grafana"
-      "RELEASE_LIB_REF": "main"
-      "RELEASE_REPO": "grafana/loki"
-    "needs":
-    - "check"
-    "runs-on": "ubuntu-latest"
-    "steps":
-    - "name": "pull release library code"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "lib"
-        "ref": "${{ env.RELEASE_LIB_REF }}"
-        "repository": "grafana/loki-release"
-    - "name": "pull code to release"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "release"
-        "repository": "${{ env.RELEASE_REPO }}"
-    - "name": "setup node"
-      "uses": "actions/setup-node@v4"
-      "with":
-        "node-version": 20
-    - "name": "Set up QEMU"
-      "uses": "docker/setup-qemu-action@v3"
-    - "name": "set up docker buildx"
-      "uses": "docker/setup-buildx-action@v3"
-    - "name": "Login to DockerHub (from vault)"
-      "uses": "grafana/shared-workflows/actions/dockerhub-login@main"
-    - "id": "weekly-version"
-      "name": "Get weekly version"
-      "run": |
-        echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT
-      "working-directory": "release"
-    - "name": "Build and push"
-      "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}"
-      "uses": "docker/build-push-action@v6"
-      "with":
-        "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}"
-        "context": "release"
-        "file": "release/clients/cmd/fluent-bit/Dockerfile"
-        "platforms": "linux/amd64"
-        "push": true
-        "tags": "${{ env.IMAGE_PREFIX }}/fluent-bit-plugin-loki:${{ steps.weekly-version.outputs.version }}"
-  "fluentd":
-    "env":
-      "BUILD_TIMEOUT": 60
-      "IMAGE_PREFIX": "grafana"
-      "RELEASE_LIB_REF": "main"
-      "RELEASE_REPO": "grafana/loki"
-    "needs":
-    - "check"
-    "runs-on": "ubuntu-latest"
-    "steps":
-    - "name": "pull release library code"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "lib"
-        "ref": "${{ env.RELEASE_LIB_REF }}"
-        "repository": "grafana/loki-release"
-    - "name": "pull code to release"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "release"
-        "repository": "${{ env.RELEASE_REPO }}"
-    - "name": "setup node"
-      "uses": "actions/setup-node@v4"
-      "with":
-        "node-version": 20
-    - "name": "Set up QEMU"
-      "uses": "docker/setup-qemu-action@v3"
-    - "name": "set up docker buildx"
-      "uses": "docker/setup-buildx-action@v3"
-    - "name": "Login to DockerHub (from vault)"
-      "uses": "grafana/shared-workflows/actions/dockerhub-login@main"
-    - "id": "weekly-version"
-      "name": "Get weekly version"
-      "run": |
-        echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT
-      "working-directory": "release"
-    - "name": "Build and push"
-      "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}"
-      "uses": "docker/build-push-action@v6"
-      "with":
-        "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}"
-        "context": "release"
-        "file": "release/clients/cmd/fluentd/Dockerfile"
-        "platforms": "linux/amd64"
-        "push": true
-        "tags": "${{ env.IMAGE_PREFIX }}/fluent-plugin-loki:${{ steps.weekly-version.outputs.version }}"
-  "logcli":
-    "env":
-      "BUILD_TIMEOUT": 60
-      "IMAGE_PREFIX": "grafana"
-      "RELEASE_LIB_REF": "main"
-      "RELEASE_REPO": "grafana/loki"
-    "needs":
-    - "check"
-    "runs-on": "ubuntu-latest"
-    "steps":
-    - "name": "pull release library code"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "lib"
-        "ref": "${{ env.RELEASE_LIB_REF }}"
-        "repository": "grafana/loki-release"
-    - "name": "pull code to release"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "release"
-        "repository": "${{ env.RELEASE_REPO }}"
-    - "name": "setup node"
-      "uses": "actions/setup-node@v4"
-      "with":
-        "node-version": 20
-    - "name": "Set up QEMU"
-      "uses": "docker/setup-qemu-action@v3"
-    - "name": "set up docker buildx"
-      "uses": "docker/setup-buildx-action@v3"
-    - "name": "Login to DockerHub (from vault)"
-      "uses": "grafana/shared-workflows/actions/dockerhub-login@main"
-    - "id": "weekly-version"
-      "name": "Get weekly version"
-      "run": |
-        echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT
-      "working-directory": "release"
-    - "name": "Build and push"
-      "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}"
-      "uses": "docker/build-push-action@v6"
-      "with":
-        "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}"
-        "context": "release"
-        "file": "release/cmd/logcli/Dockerfile"
-        "platforms": "linux/amd64,linux/arm64,linux/arm"
-        "push": true
-        "tags": "${{ env.IMAGE_PREFIX }}/logcli:${{ steps.weekly-version.outputs.version }}"
-  "logstash":
-    "env":
-      "BUILD_TIMEOUT": 60
-      "IMAGE_PREFIX": "grafana"
-      "RELEASE_LIB_REF": "main"
-      "RELEASE_REPO": "grafana/loki"
-    "needs":
-    - "check"
-    "runs-on": "ubuntu-latest"
-    "steps":
-    - "name": "pull release library code"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "lib"
-        "ref": "${{ env.RELEASE_LIB_REF }}"
-        "repository": "grafana/loki-release"
-    - "name": "pull code to release"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "release"
-        "repository": "${{ env.RELEASE_REPO }}"
-    - "name": "setup node"
-      "uses": "actions/setup-node@v4"
-      "with":
-        "node-version": 20
-    - "name": "Set up QEMU"
-      "uses": "docker/setup-qemu-action@v3"
-    - "name": "set up docker buildx"
-      "uses": "docker/setup-buildx-action@v3"
-    - "name": "Login to DockerHub (from vault)"
-      "uses": "grafana/shared-workflows/actions/dockerhub-login@main"
-    - "id": "weekly-version"
-      "name": "Get weekly version"
-      "run": |
-        echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT
-      "working-directory": "release"
-    - "name": "Build and push"
-      "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}"
-      "uses": "docker/build-push-action@v6"
-      "with":
-        "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}"
-        "context": "release"
-        "file": "release/clients/cmd/logstash/Dockerfile"
-        "platforms": "linux/amd64"
-        "push": true
-        "tags": "${{ env.IMAGE_PREFIX }}/logstash-output-loki:${{ steps.weekly-version.outputs.version }}"
   "loki":
     "env":
       "BUILD_TIMEOUT": 60
@@ -329,78 +145,6 @@
         "platforms": "linux/amd64,linux/arm64,linux/arm"
         "push": true
         "tags": "${{ env.IMAGE_PREFIX }}/loki-canary-boringcrypto:${{ steps.weekly-version.outputs.version }}"
-  "loki-docker-driver":
-    "env":
-      "BUILD_TIMEOUT": 60
-      "IMAGE_PREFIX": "grafana"
-      "RELEASE_LIB_REF": "main"
-      "RELEASE_REPO": "grafana/loki"
-    "needs":
-    - "check"
-    "runs-on": "ubuntu-latest"
-    "steps":
-    - "name": "pull release library code"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "lib"
-        "ref": "${{ env.RELEASE_LIB_REF }}"
-        "repository": "grafana/loki-release"
-    - "name": "pull code to release"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "release"
-        "repository": "${{ env.RELEASE_REPO }}"
-    - "name": "setup node"
-      "uses": "actions/setup-node@v4"
-      "with":
-        "node-version": 20
-    - "name": "Set up QEMU"
-      "uses": "docker/setup-qemu-action@v3"
-    - "name": "set up docker buildx"
-      "uses": "docker/setup-buildx-action@v3"
-    - "name": "Login to DockerHub (from vault)"
-      "uses": "grafana/shared-workflows/actions/dockerhub-login@main"
-    - "id": "weekly-version"
-      "name": "Get weekly version"
-      "run": |
-        echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT
-        
-        platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
-        echo "platform=${platform}" >> $GITHUB_OUTPUT
-        echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
-        if [[ "${platform}" == "linux/arm64" ]]; then
-          echo "plugin_arch=-arm64" >> $GITHUB_OUTPUT
-        else
-          echo "plugin_arch=" >> $GITHUB_OUTPUT
-        fi
-      "working-directory": "release"
-    - "name": "Build and export"
-      "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}"
-      "uses": "docker/build-push-action@v6"
-      "with":
-        "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }},GOARCH=${{ steps.weekly-version.outputs.platform_short }}"
-        "context": "release"
-        "file": "release/clients/cmd/docker-driver/Dockerfile"
-        "outputs": "type=docker,dest=release/images/grafana/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar"
-        "platforms": "${{ matrix.platform }}"
-        "push": false
-        "tags": "${{ env.IMAGE_PREFIX }}/grafana/loki-docker-driver:${{ steps.weekly-version.outputs.version }}"
-    - "env":
-        "BUILD_DIR": "release/clients/cmd/docker-driver"
-        "IMAGE_TAG": "${{ steps.weekly-version.outputs.version }}"
-      "name": "Package and push as Docker plugin"
-      "run": |
-        rm -rf "${{ env.BUILD_DIR }}/rootfs" || true
-        mkdir "${{ env.BUILD_DIR }}/rootfs"
-        tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar"
-        docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}"
-        docker plugin push "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}"
-      "working-directory": "release"
-    "strategy":
-      "matrix":
-        "platform":
-        - "linux/amd64"
-        - "linux/arm64"
   "promtail":
     "env":
       "BUILD_TIMEOUT": 60
@@ -447,52 +191,6 @@
         "platforms": "linux/amd64,linux/arm64,linux/arm"
         "push": true
         "tags": "${{ env.IMAGE_PREFIX }}/promtail:${{ steps.weekly-version.outputs.version }}"
-  "querytee":
-    "env":
-      "BUILD_TIMEOUT": 60
-      "IMAGE_PREFIX": "grafana"
-      "RELEASE_LIB_REF": "main"
-      "RELEASE_REPO": "grafana/loki"
-    "needs":
-    - "check"
-    "runs-on": "ubuntu-latest"
-    "steps":
-    - "name": "pull release library code"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "lib"
-        "ref": "${{ env.RELEASE_LIB_REF }}"
-        "repository": "grafana/loki-release"
-    - "name": "pull code to release"
-      "uses": "actions/checkout@v4"
-      "with":
-        "path": "release"
-        "repository": "${{ env.RELEASE_REPO }}"
-    - "name": "setup node"
-      "uses": "actions/setup-node@v4"
-      "with":
-        "node-version": 20
-    - "name": "Set up QEMU"
-      "uses": "docker/setup-qemu-action@v3"
-    - "name": "set up docker buildx"
-      "uses": "docker/setup-buildx-action@v3"
-    - "name": "Login to DockerHub (from vault)"
-      "uses": "grafana/shared-workflows/actions/dockerhub-login@main"
-    - "id": "weekly-version"
-      "name": "Get weekly version"
-      "run": |
-        echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT
-      "working-directory": "release"
-    - "name": "Build and push"
-      "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}"
-      "uses": "docker/build-push-action@v6"
-      "with":
-        "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}"
-        "context": "release"
-        "file": "release/cmd/querytee/Dockerfile"
-        "platforms": "linux/amd64"
-        "push": true
-        "tags": "${{ env.IMAGE_PREFIX }}/loki-query-tee:${{ steps.weekly-version.outputs.version }}"
 "name": "publish images"
 "on":
   "push":

From c96b750a3f6bee9a76482ead8823e71405e5d68b Mon Sep 17 00:00:00 2001
From: Trevor Whitney <trevorjwhitney@gmail.com>
Date: Mon, 9 Dec 2024 11:46:12 -0700
Subject: [PATCH 12/14] fix: enable metric aggregation in
 `loki-local-config.yaml` (#15302)

---
 cmd/loki/loki-local-config.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cmd/loki/loki-local-config.yaml b/cmd/loki/loki-local-config.yaml
index 2df5f0f931f7b..4aff8772ae1da 100644
--- a/cmd/loki/loki-local-config.yaml
+++ b/cmd/loki/loki-local-config.yaml
@@ -25,6 +25,9 @@ query_range:
         enabled: true
         max_size_mb: 100
 
+limits_config:
+  metric_aggregation_enabled: true
+
 schema_config:
   configs:
     - from: 2020-10-24

From f5d62bd505c19ceb287bbcb65359156f7273e75f Mon Sep 17 00:00:00 2001
From: Trevor Whitney <trevorjwhitney@gmail.com>
Date: Mon, 9 Dec 2024 11:46:29 -0700
Subject: [PATCH 13/14] feat: present DF bytes values in queryable format
 (#15272)

---
 pkg/querier/queryrange/detected_fields.go     |  9 ++-
 .../queryrange/detected_fields_test.go        | 66 ++++++++++++++++++-
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/pkg/querier/queryrange/detected_fields.go b/pkg/querier/queryrange/detected_fields.go
index 3248d3b2cda81..be42d1f0a6868 100644
--- a/pkg/querier/queryrange/detected_fields.go
+++ b/pkg/querier/queryrange/detected_fields.go
@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"slices"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/axiomhq/hyperloglog"
@@ -114,7 +115,13 @@ func parseDetectedFieldValues(limit uint32, streams []push.Stream, name string)
 			parsedLabels, _ := parseEntry(entry, entryLbls)
 			if vals, ok := parsedLabels[name]; ok {
 				for _, v := range vals {
-					values[v] = struct{}{}
+					// special case bytes values, so they can be directly inserted into a query
+					if bs, err := humanize.ParseBytes(v); err == nil {
+						bsString := strings.Replace(humanize.Bytes(bs), " ", "", 1)
+						values[bsString] = struct{}{}
+					} else {
+						values[v] = struct{}{}
+					}
 				}
 			}
 		}
diff --git a/pkg/querier/queryrange/detected_fields_test.go b/pkg/querier/queryrange/detected_fields_test.go
index b0b363e4735d1..22aa63e284888 100644
--- a/pkg/querier/queryrange/detected_fields_test.go
+++ b/pkg/querier/queryrange/detected_fields_test.go
@@ -24,7 +24,7 @@ import (
 	"github.com/grafana/loki/pkg/push"
 )
 
-func Test_parseDetectedFeilds(t *testing.T) {
+func Test_parseDetectedFields(t *testing.T) {
 	now := time.Now()
 
 	t.Run("when no parsers are supplied", func(t *testing.T) {
@@ -1317,6 +1317,70 @@ func TestQuerier_DetectedFields(t *testing.T) {
 			}, secondValues)
 		},
 	)
+
+	t.Run("correctly formats bytes values for detected fields", func(t *testing.T) {
+		lbls := `{cluster="us-east-1", namespace="mimir-dev", pod="mimir-ruler-nfb37", service_name="mimir-ruler"}`
+		metric, err := parser.ParseMetric(lbls)
+		require.NoError(t, err)
+		now := time.Now()
+
+		infoDetectdFiledMetadata := []push.LabelAdapter{
+			{
+				Name:  "detected_level",
+				Value: "info",
+			},
+		}
+
+		lines := []push.Entry{
+			{
+				Timestamp:          now,
+				Line:               "ts=2024-09-05T15:36:38.757788067Z caller=metrics.go:66 tenant=2419 level=info bytes=1,024",
+				StructuredMetadata: infoDetectdFiledMetadata,
+			},
+			{
+				Timestamp:          now,
+				Line:               `ts=2024-09-05T15:36:38.698375619Z caller=grpc_logging.go:66 tenant=29 level=info bytes="1024 MB"`,
+				StructuredMetadata: infoDetectdFiledMetadata,
+			},
+			{
+				Timestamp:          now,
+				Line:               "ts=2024-09-05T15:36:38.629424175Z caller=grpc_logging.go:66 tenant=2919 level=info bytes=1024KB",
+				StructuredMetadata: infoDetectdFiledMetadata,
+			},
+		}
+		stream := push.Stream{
+			Labels:  lbls,
+			Entries: lines,
+			Hash:    metric.Hash(),
+		}
+
+		handler := NewDetectedFieldsHandler(
+			limitedHandler(stream),
+			logHandler(stream),
+			limits,
+		)
+
+		request := DetectedFieldsRequest{
+			logproto.DetectedFieldsRequest{
+				Start:     time.Now().Add(-1 * time.Minute),
+				End:       time.Now(),
+				Query:     `{cluster="us-east-1"} | logfmt`,
+				LineLimit: 1000,
+				Limit:     3,
+				Values:    true,
+				Name:      "bytes",
+			},
+			"/loki/api/v1/detected_field/bytes/values",
+		}
+
+		detectedFieldValues := handleRequest(handler, request).Values
+		slices.Sort(detectedFieldValues)
+		require.Equal(t, []string{
+			"1.0GB",
+			"1.0MB",
+			"1.0kB",
+		}, detectedFieldValues)
+	})
 }
 
 func BenchmarkQuerierDetectedFields(b *testing.B) {

From 03a05876975d5367f9b10cf10e9506b09f01e70b Mon Sep 17 00:00:00 2001
From: Trevor Whitney <trevorjwhitney@gmail.com>
Date: Mon, 9 Dec 2024 13:46:44 -0700
Subject: [PATCH 14/14] ci: fix nix build (#15308)

---
 .github/workflows/nix-ci.yaml |   3 +
 CODEOWNERS                    |   4 +
 flake.nix                     |  81 +++++++--------
 nix/default.nix               | 179 +++++++++++++++++-----------------
 nix/packages/faillint.nix     |  16 ---
 nix/packages/loki.nix         |  98 ++++++-------------
 pkg/loki/loki_test.go         |   1 +
 pkg/loki/modules_test.go      |   1 +
 8 files changed, 162 insertions(+), 221 deletions(-)
 delete mode 100644 nix/packages/faillint.nix

diff --git a/.github/workflows/nix-ci.yaml b/.github/workflows/nix-ci.yaml
index ba08c719ee3dc..c76e45a199495 100644
--- a/.github/workflows/nix-ci.yaml
+++ b/.github/workflows/nix-ci.yaml
@@ -1,6 +1,9 @@
 ---
 name: "Lint And Build Nix Flake"
 on:
+  push:
+    branches:
+      - main
   pull_request:
     paths:
       - "flake.nix"
diff --git a/CODEOWNERS b/CODEOWNERS
index a118c5a40c0b1..1d122e1f580c5 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -12,5 +12,9 @@
 # The observability logs team is listed as co-codeowner for grammar file. This is to receive notifications about updates, so these can be implemented in https://github.com/grafana/lezer-logql
 /pkg/logql/syntax/expr.y @grafana/observability-logs @grafana/loki-team
 
+# Nix
+/nix/ @trevorwhitney
+flake.nix @trevorwhitney
+
 # No owners - allows sub-maintainers to merge changes.
 CHANGELOG.md
diff --git a/flake.nix b/flake.nix
index c9e52eb589af3..32642f11d110d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -6,41 +6,21 @@
     flake-utils.url = "github:numtide/flake-utils";
   };
 
-  # Nixpkgs / NixOS version to use.
-
   outputs = { self, nixpkgs, flake-utils }:
-    let
-      nix = import ./nix { inherit self; };
-    in
-    {
-      overlays = {
-        default = nix.overlay;
-      };
-    } //
     flake-utils.lib.eachDefaultSystem (system:
       let
-        pkgs = import nixpkgs {
-          inherit system;
-          overlays = [
-            nix.overlay
-          ];
-          config = { allowUnfree = true; };
-        };
+        pkgs = import nixpkgs
+          {
+            inherit system;
+            config = { allowUnfree = true; };
+          };
       in
-      {
-        # The default package for 'nix build'. This makes sense if the
-        # flake provides only one package or there is a clear "main"
-        # package.
+      rec {
         defaultPackage = pkgs.loki;
 
-        packages = with pkgs; {
-          inherit
-            logcli
-            loki
-            loki-canary
-            loki-helm-test
-            loki-helm-test-docker
-            promtail;
+        packages = import ./nix {
+          inherit self pkgs;
+          inherit (pkgs) lib;
         };
 
         apps = {
@@ -56,18 +36,31 @@
 
           test = {
             type = "app";
-            program = with pkgs; "${
-                (writeShellScriptBin "test.sh" ''
-                  ${loki.overrideAttrs(old: { 
-                  buildInputs =
-                    let
-                      inherit (old) buildInputs;
-                    in
-                    if pkgs.stdenv.hostPlatform.isLinux then
-                      buildInputs ++ (with pkgs; [ systemd ])
-                    else buildInputs;
-                  doCheck = true; 
-                  })}/bin/loki --version
+            program =
+              let
+                loki = packages.loki.overrideAttrs (old: {
+                  buildInputs = with pkgs; lib.optionals stdenv.hostPlatform.isLinux [ systemd.dev ];
+                  doCheck = true;
+                  checkFlags = [
+                    "-covermode=atomic"
+                    "-coverprofile=coverage.txt"
+                    "-p=4"
+                  ];
+                  subPackages = [
+                    "./..." # for tests
+                    "cmd/loki"
+                    "cmd/logcli"
+                    "cmd/loki-canary"
+                    "clients/cmd/promtail"
+                  ];
+                });
+              in
+              "${
+                (pkgs.writeShellScriptBin "test.sh" ''
+                  ${loki}/bin/loki --version
+                  ${loki}/bin/logcli --version
+                  ${loki}/bin/loki-canary --version
+                  ${loki}/bin/promtail --version
                 '')
               }/bin/test.sh";
           };
@@ -80,10 +73,6 @@
               inherit (pkgs) buildGoModule fetchFromGitHub;
             })
 
-            (pkgs.callPackage ./nix/packages/faillint.nix {
-              inherit (pkgs) lib buildGoModule fetchFromGitHub;
-            })
-
             chart-testing
             gcc
             go
@@ -94,7 +83,7 @@
             nixpkgs-fmt
             statix
             yamllint
-          ];
+          ] // packages;
         };
       });
 }
diff --git a/nix/default.nix b/nix/default.nix
index d2c45ae2af383..1b00605962cb8 100644
--- a/nix/default.nix
+++ b/nix/default.nix
@@ -1,92 +1,91 @@
-{ self }:
+{ self, pkgs, lib }:
+let
+  # self.rev is only set on a clean git tree
+  gitRevision = if (self ? rev) then self.rev else "dirty";
+  shortGitRevsion = with lib;
+    if (self ? rev) then
+      (strings.concatStrings
+        (lists.take 8 (strings.stringToCharacters gitRevision)))
+    else
+      "dirty";
+
+  # the image tag script is hard coded to take only 7 characters
+  imageTagVersion = with lib;
+    if (self ? rev) then
+      (strings.concatStrings
+        (lists.take 8 (strings.stringToCharacters gitRevision)))
+    else
+      "dirty";
+
+  imageTag =
+    if (self ? rev) then
+      "${imageTagVersion}"
+    else
+      "${imageTagVersion}-WIP";
+
+  meta = with lib; {
+    homepage = "https://grafana.com/oss/loki/";
+    changelog = "https://github.com/grafana/loki/commit/${shortGitRevsion}";
+    maintainers = with maintainers; [ trevorwhitney ];
+
+  };
+
+  loki-helm-test = pkgs.callPackage ../production/helm/loki/src/helm-test {
+    inherit pkgs;
+    inherit (pkgs) lib buildGoModule dockerTools;
+    rev = gitRevision;
+  };
+in
 {
-  overlay = final: prev:
-    let
-      # self.rev is only set on a clean git tree
-      gitRevision = if (self ? rev) then self.rev else "dirty";
-      shortGitRevsion = with prev.lib;
-        if (self ? rev) then
-          (strings.concatStrings
-            (lists.take 8 (strings.stringToCharacters gitRevision)))
-        else
-          "dirty";
-
-      # the image tag script is hard coded to take only 7 characters
-      imageTagVersion = with prev.lib;
-        if (self ? rev) then
-          (strings.concatStrings
-            (lists.take 8 (strings.stringToCharacters gitRevision)))
-        else
-          "dirty";
-
-      imageTag =
-        if (self ? rev) then
-          "${imageTagVersion}"
-        else
-          "${imageTagVersion}-WIP";
-
-      loki-helm-test = prev.callPackage ../production/helm/loki/src/helm-test {
-        inherit (prev) pkgs lib buildGoModule dockerTools;
-        rev = gitRevision;
-      };
-    in
-    {
-      inherit (loki-helm-test) loki-helm-test loki-helm-test-docker;
-    } // rec {
-      loki = prev.callPackage ./packages/loki.nix {
-        inherit imageTag;
-        version = shortGitRevsion;
-        pkgs = prev;
-      };
-
-      logcli = loki.overrideAttrs (oldAttrs: {
-        pname = "logcli";
-
-        buildPhase = ''
-          export GOCACHE=$TMPDIR/go-cache
-          make clean logcli
-        '';
-
-        installPhase = ''
-          mkdir -p $out/bin
-          install -m755 cmd/logcli/logcli $out/bin/logcli
-        '';
-      });
-
-      loki-canary = loki.overrideAttrs (oldAttrs: {
-        pname = "loki-canary";
-
-        buildPhase = ''
-          export GOCACHE=$TMPDIR/go-cache
-          make clean loki-canary
-        '';
-
-        installPhase = ''
-          mkdir -p $out/bin
-          install -m755 cmd/loki-canary/loki-canary $out/bin/loki-canary
-        '';
-      });
-
-      promtail = loki.overrideAttrs (oldAttrs: {
-        pname = "promtail";
-
-        buildInputs =
-          let
-            inherit (oldAttrs) buildInputs;
-          in
-          if prev.stdenv.hostPlatform.isLinux then
-            buildInputs ++ (with prev; [ systemd ])
-          else buildInputs;
-
-        buildPhase = ''
-          export GOCACHE=$TMPDIR/go-cache
-          make clean promtail
-        '';
-
-        installPhase = ''
-          mkdir -p $out/bin
-          install -m755 clients/cmd/promtail/promtail $out/bin/promtail
-        '';
-      });
-    };
+  inherit (loki-helm-test) loki-helm-test loki-helm-test-docker;
+} // rec {
+  loki = pkgs.callPackage ./packages/loki.nix {
+    inherit imageTag pkgs;
+    version = shortGitRevsion;
+  };
+
+  logcli = loki.overrideAttrs (oldAttrs: {
+    pname = "logcli";
+
+    subPackages = [ "cmd/logcli" ];
+
+    meta = with lib; {
+      description = "LogCLI is a command line tool for interacting with Loki.";
+      mainProgram = "logcli";
+      license = with licenses; [ agpl3Only ];
+    } // meta;
+  });
+
+  loki-canary = loki.overrideAttrs (oldAttrs: {
+    pname = "loki-canary";
+
+    subPackages = [ "cmd/loki-canary" ];
+
+    meta = with lib; {
+      description = "Loki Canary is a canary for the Loki project.";
+      mainProgram = "loki-canary";
+      license = with licenses; [ agpl3Only ];
+    } // meta;
+  });
+
+  promtail = loki.overrideAttrs (oldAttrs: {
+    pname = "promtail";
+
+    buildInputs = with pkgs; lib.optionals stdenv.hostPlatform.isLinux [ systemd.dev ];
+
+    tags = [ "promtail_journal_enabled" ];
+
+    subPackages = [ "clients/cmd/promtail" ];
+
+    preFixup = lib.optionalString pkgs.stdenv.hostPlatform.isLinux ''
+      wrapProgram $out/bin/promtail \
+        --prefix LD_LIBRARY_PATH : "${lib.getLib pkgs.systemd}/lib"
+    '';
+
+    meta = with lib; {
+      description = "Client for sending logs to Loki";
+      mainProgram = "promtail";
+      license = with licenses; [ asl20 ];
+    } // meta;
+  });
 }
diff --git a/nix/packages/faillint.nix b/nix/packages/faillint.nix
deleted file mode 100644
index 0931fc08fccfa..0000000000000
--- a/nix/packages/faillint.nix
+++ /dev/null
@@ -1,16 +0,0 @@
-{ lib, buildGoModule, fetchFromGitHub }:
-
-buildGoModule rec {
-  pname = "faillint";
-  version = "v1.14.0";
-
-  src = fetchFromGitHub {
-    owner = "fatih";
-    repo = "faillint";
-    rev = "${version}";
-    sha256 = "NV+wbu547mtTa6dTGv7poBwWXOmu5YjqbauzolCg5qs=";
-  };
-
-  vendorHash = "sha256-vWt4HneDA7YwXYnn8TbfWCKzSv7RcgXxn/HAh6a+htQ=";
-  doCheck = false;
-}
diff --git a/nix/packages/loki.nix b/nix/packages/loki.nix
index 7da43f05d3ede..28f215155b36a 100644
--- a/nix/packages/loki.nix
+++ b/nix/packages/loki.nix
@@ -1,78 +1,38 @@
-{ pkgs, version, imageTag }:
-let
-  lambda-promtail-gomod = pkgs.buildGoModule {
-    inherit version;
-    pname = "lambda-promtail";
-
-    src = ./../../tools/lambda-promtail;
-    vendorHash = "sha256-yQIRFUhod91HiPS5IKm7eNeIXJzBWVcvIXf9qMncTKw=";
-
-    doCheck = false;
-
-    installPhase = ''
-      runHook preInstall
-      cp -r --reflink=auto vendor $out
-      runHook postInstall
-    '';
-  };
-in
-pkgs.stdenv.mkDerivation {
+{ pkgs, version, imageTag, lib }:
+pkgs.buildGo123Module {
   inherit version;
 
   pname = "loki";
 
   src = ./../..;
-
-  buildInputs = with pkgs; [
-    bash
-    gcc
-    git
-    go_1_23
-    golangci-lint
-    gotools
-    nettools
-    yamllint
-
-    (import ./faillint.nix {
-      inherit (pkgs) lib buildGoModule fetchFromGitHub;
-    })
-  ];
-
-  configurePhase = with pkgs; ''
-    patchShebangs tools
-
-    substituteInPlace Makefile \
-      --replace "SHELL = /usr/bin/env bash -o pipefail" "SHELL = ${bash}/bin/bash -o pipefail" \
-      --replace "IMAGE_TAG ?= \$(shell ./tools/image-tag)" "IMAGE_TAG ?= ${imageTag}" \
-      --replace "GIT_REVISION := \$(shell git rev-parse --short HEAD)" "GIT_REVISION := ${version}" \
-      --replace "GIT_BRANCH := \$(shell git rev-parse --abbrev-ref HEAD)" "GIT_BRANCH := nix"
-
-    substituteInPlace clients/cmd/fluentd/Makefile \
-      --replace "SHELL    = /usr/bin/env bash -o pipefail" "SHELL = ${bash}/bin/bash -o pipefail"
-  '';
-
-  buildPhase = ''
-    export GOCACHE=$TMPDIR/go-cache
-    export GOMODCACHE=$TMPDIR/gomodcache
-    export GOPROXY=off
-
-    cp -r ${lambda-promtail-gomod} tools/lambda-promtail/vendor
-    make clean loki
-  '';
+  vendorHash = null;
+
+  ldflags =
+    let
+      prefix = "github.com/grafana/loki/v3/pkg/util/build";
+    in
+    [
+      "-s"
+      "-w"
+      "-X ${prefix}.Branch=nix"
+      "-X ${prefix}.Version=${imageTag}"
+      "-X ${prefix}.Revision=${version}"
+      "-X ${prefix}.BuildUser=nix@nixpkgs"
+      "-X ${prefix}.BuildDate=unknown"
+    ];
+
+  subPackages = [ "cmd/loki" ];
+
+  nativeBuildInputs = with pkgs; [ makeWrapper ];
 
   doCheck = false;
-  checkPhase = ''
-    export GOCACHE=$TMPDIR/go-cache
-    export GOMODCACHE=$TMPDIR/gomodcache
-    export GOLANGCI_LINT_CACHE=$TMPDIR/go-cache
-    export GOPROXY=off
-    export BUILD_IN_CONTAINER=false
-
-    make lint test
-  '';
 
-  installPhase = ''
-    mkdir -p $out/bin
-    install -m755 cmd/loki/loki $out/bin/loki
-  '';
+  meta = with lib; {
+    description = "Like Prometheus, but for logs";
+    mainProgram = "loki";
+    license = with licenses; [ agpl3Only ];
+    homepage = "https://grafana.com/oss/loki/";
+    changelog = "https://github.com/grafana/loki/commit/${version}";
+    maintainers = with maintainers; [ trevorwhitney ];
+  };
 }
diff --git a/pkg/loki/loki_test.go b/pkg/loki/loki_test.go
index b29d2aad22065..3d5f774247754 100644
--- a/pkg/loki/loki_test.go
+++ b/pkg/loki/loki_test.go
@@ -188,6 +188,7 @@ server:
 common:
   compactor_address: http://localhost:%d
   path_prefix: /tmp/loki
+  instance_addr: localhost
   ring:
     kvstore:
       store: inmemory
diff --git a/pkg/loki/modules_test.go b/pkg/loki/modules_test.go
index 64241443d3439..f3b9fa9a98d51 100644
--- a/pkg/loki/modules_test.go
+++ b/pkg/loki/modules_test.go
@@ -406,6 +406,7 @@ func minimalWorkingConfig(t *testing.T, dir, target string, cfgTransformers ...f
 	}
 
 	cfg.Common.InstanceAddr = localhost
+	cfg.MemberlistKV.AdvertiseAddr = localhost
 	cfg.Ingester.LifecyclerConfig.Addr = localhost
 	cfg.Distributor.DistributorRing.InstanceAddr = localhost
 	cfg.IndexGateway.Mode = indexgateway.SimpleMode