Merge remote-tracking branch 'upstream/main' into blooms/metas-and-ex…

…pired-removals

Signed-off-by: Owen Diehl <[email protected]>

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ##### Enhancements
 
+* [11851](https://github.com/grafana/loki/pull/11851) **elcomtik**: Helm: Allow the definition of resources for GrafanaAgent pods.
+* [11819](https://github.com/grafana/loki/pull/11819) **jburnham**: Ruler: Add the ability to disable the `X-Scope-OrgId` tenant identification header in remote write requests.
 * [11633](https://github.com/grafana/loki/pull/11633) **cyriltovena**: Add profiling integrations to tracing instrumentation.
 * [11571](https://github.com/grafana/loki/pull/11571) **MichelHollands**: Add a metrics.go log line for requests from querier to ingester
 * [11477](https://github.com/grafana/loki/pull/11477) **MichelHollands**: support GET for /ingester/shutdown
@@ -54,6 +56,7 @@
 * [11143](https://github.com/grafana/loki/pull/11143) **sandeepsukhani** otel: Add support for per tenant configuration for mapping otlp data to loki format
 * [11499](https://github.com/grafana/loki/pull/11284) **jmichalek132** Config: Adds `frontend.log-query-request-headers` to enable logging of request headers in query logs.
 * [11817](https://github.com/grafana/loki/pull/11817) **ashwanthgoli** Ruler: Add support for filtering results of `/prometheus/api/v1/rules` endpoint by rule_name, rule_group, file and type.
+* [11897](https://github.com/grafana/loki/pull/11897) **ashwanthgoli** Metadata: Introduces a separate split interval of `split_recent_metadata_queries_by_interval` for `recent_metadata_query_window` to help with caching recent metadata query results.
 
 ##### Fixes
 * [11074](https://github.com/grafana/loki/pull/11074) **hainenber** Fix panic in lambda-promtail due to mishandling of empty DROP_LABELS env var.

docs/sources/configure/_index.md

@@ -1274,6 +1274,10 @@ remote_write:
   # CLI flag: -ruler.remote-write.config-refresh-period
   [config_refresh_period: <duration> | default = 10s]
 
+  # Add X-Scope-OrgID header in remote write requests.
+  # CLI flag: -ruler.remote-write.add-org-id-header
+  [add_org_id_header: <boolean> | default = true]
+
 # Configuration for rule evaluation.
 evaluation:
   # The evaluation mode for the ruler. Can be either 'local' or 'remote'. If set
@@ -2907,6 +2911,30 @@ The `limits_config` block configures global and per-tenant limits in Loki.
 # CLI flag: -querier.split-metadata-queries-by-interval
 [split_metadata_queries_by_interval: <duration> | default = 1d]
 
+# Experimental. Split interval to use for the portion of metadata request that
+# falls within `recent_metadata_query_window`. Rest of the request which is
+# outside the window still uses `split_metadata_queries_by_interval`. If set to
+# 0, the entire request defaults to using a split interval of
+# `split_metadata_queries_by_interval.`.
+# CLI flag: -experimental.querier.split-recent-metadata-queries-by-interval
+[split_recent_metadata_queries_by_interval: <duration> | default = 1h]
+
+# Experimental. Metadata query window inside which
+# `split_recent_metadata_queries_by_interval` gets applied, portion of the
+# metadata request that falls in this window is split using
+# `split_recent_metadata_queries_by_interval`. The value 0 disables using a
+# different split interval for recent metadata queries.
+# 
+# This is added to improve cacheability of recent metadata queries. Query split
+# interval also determines the interval used in cache key. The default split
+# interval of 24h is useful for caching long queries, each cache key holding 1
+# day's results. But metadata queries are often shorter than 24h, to cache them
+# effectively we need a smaller split interval. `recent_metadata_query_window`
+# along with `split_recent_metadata_queries_by_interval` help configure a
+# shorter split interval for recent metadata queries.
+# CLI flag: -experimental.querier.recent-metadata-query-window
+[recent_metadata_query_window: <duration> | default = 0s]
+
 # Interval to use for time-based splitting when a request is within the
 # `query_ingesters_within` window; defaults to `split-queries-by-interval` by
 # setting to 0.

docs/sources/setup/install/helm/reference.md

@@ -2806,6 +2806,15 @@ true
 			<td><pre lang="json">
 null
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>monitoring.selfMonitoring.grafanaAgent.resources</td>
+			<td>object</td>
+			<td>Resource requests and limits for the grafanaAgent pods</td>
+			<td><pre lang="json">
+{}
+</pre>
 </td>
 		</tr>
 		<tr>

docs/variables.mk

@@ -1,8 +1,5 @@
 # List of projects to provide to the make-docs script.
 PROJECTS := loki
 
-# Use alternative image until make-docs 3.0.0 is rolled out.
-export DOCS_IMAGE := grafana/docs-base:dbd975af06
-
 # Set the DOC_VALIDATOR_IMAGE to match the one defined in CI.
 export DOC_VALIDATOR_IMAGE := $(shell sed -En 's, *image: "(grafana/doc-validator.*)",\1,p' "$(shell git rev-parse --show-toplevel)/.github/workflows/doc-validator.yml")

pkg/bloomcompactor/bloomcompactor.go

@@ -169,11 +169,11 @@ func runWithRetries(
 
 type tenantTable struct {
 	tenant         string
-	table          DayTable
+	table          config.DayTime
 	ownershipRange v1.FingerprintBounds
 }
 
-func (c *Compactor) tenants(ctx context.Context, table DayTable) (v1.Iterator[string], error) {
+func (c *Compactor) tenants(ctx context.Context, table config.DayTime) (v1.Iterator[string], error) {
 	tenants, err := c.tsdbStore.UsersForPeriod(ctx, table)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting tenants")
@@ -214,10 +214,9 @@ func (c *Compactor) tables(ts time.Time) *dayRangeIterator {
 	from := ts.Add(-maxCompactionPeriod).UnixNano() / int64(config.ObjectStorageIndexRequiredPeriod) * int64(config.ObjectStorageIndexRequiredPeriod)
 	through := ts.Add(-minCompactionPeriod).UnixNano() / int64(config.ObjectStorageIndexRequiredPeriod) * int64(config.ObjectStorageIndexRequiredPeriod)
 
-	fromDay := DayTable(model.TimeFromUnixNano(from))
-	throughDay := DayTable(model.TimeFromUnixNano(through))
+	fromDay := config.NewDayTime(model.TimeFromUnixNano(from))
+	throughDay := config.NewDayTime(model.TimeFromUnixNano(through))
 	return newDayRangeIterator(fromDay, throughDay)
-
 }
 
 func (c *Compactor) loadWork(ctx context.Context, ch chan<- tenantTable) error {
@@ -295,10 +294,10 @@ func (c *Compactor) compactTenantTable(ctx context.Context, tt tenantTable) erro
 }
 
 type dayRangeIterator struct {
-	min, max, cur DayTable
+	min, max, cur config.DayTime
 }
 
-func newDayRangeIterator(min, max DayTable) *dayRangeIterator {
+func newDayRangeIterator(min, max config.DayTime) *dayRangeIterator {
 	return &dayRangeIterator{min: min, max: max, cur: min.Dec()}
 }
 
@@ -307,7 +306,7 @@ func (r *dayRangeIterator) Next() bool {
 	return r.cur.Before(r.max)
 }
 
-func (r *dayRangeIterator) At() DayTable {
+func (r *dayRangeIterator) At() config.DayTime {
 	return r.cur
 }
 

pkg/bloomcompactor/config.go

@@ -5,10 +5,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/prometheus/common/model"
-
-	"github.com/grafana/loki/pkg/storage/config"
-	"github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper"
 	"github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/downloads"
 	"github.com/grafana/loki/pkg/util/ring"
 )
@@ -70,37 +66,3 @@ type Limits interface {
 	BloomFalsePositiveRate(tenantID string) float64
 	BloomCompactorMaxBlockSize(tenantID string) int
 }
-
-// TODO(owen-d): Remove this type in favor of config.DayTime
-type DayTable model.Time
-
-func (d DayTable) String() string {
-	return fmt.Sprintf("%d", d.ModelTime().Time().UnixNano()/int64(config.ObjectStorageIndexRequiredPeriod))
-}
-
-func (d DayTable) Inc() DayTable {
-	return DayTable(d.ModelTime().Add(config.ObjectStorageIndexRequiredPeriod))
-}
-
-func (d DayTable) Dec() DayTable {
-	return DayTable(d.ModelTime().Add(-config.ObjectStorageIndexRequiredPeriod))
-}
-
-func (d DayTable) Before(other DayTable) bool {
-	return d.ModelTime().Before(model.Time(other))
-}
-
-func (d DayTable) After(other DayTable) bool {
-	return d.ModelTime().After(model.Time(other))
-}
-
-func (d DayTable) ModelTime() model.Time {
-	return model.Time(d)
-}
-
-func (d DayTable) Bounds() bloomshipper.Interval {
-	return bloomshipper.Interval{
-		Start: model.Time(d),
-		End:   model.Time(d.Inc()),
-	}
-}

pkg/bloomcompactor/controller.go

@@ -13,6 +13,7 @@ import (
 	"github.com/pkg/errors"
 
 	v1 "github.com/grafana/loki/pkg/storage/bloom/v1"
+	"github.com/grafana/loki/pkg/storage/config"
 	"github.com/grafana/loki/pkg/storage/stores/shipper/bloomshipper"
 	"github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb"
 )
@@ -68,7 +69,7 @@ Compaction works as follows, split across many functions for clarity:
 */
 func (s *SimpleBloomController) compactTenant(
 	ctx context.Context,
-	table DayTable,
+	table config.DayTime,
 	tenant string,
 	ownershipRange v1.FingerprintBounds,
 ) error {
@@ -81,15 +82,11 @@ func (s *SimpleBloomController) compactTenant(
 	}
 
 	// Fetch source metas to be used in both compaction and cleanup of out-of-date metas+blooms
-	bounds := table.Bounds()
 	metas, err := s.bloomStore.FetchMetas(
 		ctx,
 		bloomshipper.MetaSearchParams{
 			TenantID: tenant,
-			Interval: bloomshipper.Interval{
-				Start: bounds.Start,
-				End:   bounds.End,
-			},
+			Interval: bloomshipper.NewInterval(table.Bounds()),
 			Keyspace: ownershipRange,
 		},
 	)
@@ -136,10 +133,7 @@ func (s *SimpleBloomController) compactTenant(
 		ctx,
 		bloomshipper.MetaSearchParams{
 			TenantID: tenant,
-			Interval: bloomshipper.Interval{
-				Start: bounds.Start,
-				End:   bounds.End,
-			},
+			Interval: bloomshipper.NewInterval(table.Bounds()),
 			Keyspace: superset,
 		},
 	)
@@ -184,7 +178,7 @@ func (s *SimpleBloomController) compactTenant(
 func (s *SimpleBloomController) findOutdatedGaps(
 	ctx context.Context,
 	tenant string,
-	table DayTable,
+	table config.DayTime,
 	ownershipRange v1.FingerprintBounds,
 	metas []bloomshipper.Meta,
 	logger log.Logger,
@@ -224,7 +218,7 @@ func (s *SimpleBloomController) findOutdatedGaps(
 
 func (s *SimpleBloomController) loadWorkForGap(
 	ctx context.Context,
-	table DayTable,
+	table config.DayTime,
 	tenant string,
 	id tsdb.Identifier,
 	gap gapWithBlocks,
@@ -252,7 +246,7 @@ func (s *SimpleBloomController) loadWorkForGap(
 func (s *SimpleBloomController) buildGaps(
 	ctx context.Context,
 	tenant string,
-	table DayTable,
+	table config.DayTime,
 	client bloomshipper.Client,
 	work []blockPlan,
 	logger log.Logger,

pkg/bloomcompactor/tsdb.go

@@ -26,11 +26,11 @@ const (
 )
 
 type TSDBStore interface {
-	UsersForPeriod(ctx context.Context, table DayTable) ([]string, error)
-	ResolveTSDBs(ctx context.Context, table DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error)
+	UsersForPeriod(ctx context.Context, table config.DayTime) ([]string, error)
+	ResolveTSDBs(ctx context.Context, table config.DayTime, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error)
 	LoadTSDB(
 		ctx context.Context,
-		table DayTable,
+		table config.DayTime,
 		tenant string,
 		id tsdb.Identifier,
 		bounds v1.FingerprintBounds,
@@ -49,13 +49,13 @@ func NewBloomTSDBStore(storage storage.Client) *BloomTSDBStore {
 	}
 }
 
-func (b *BloomTSDBStore) UsersForPeriod(ctx context.Context, table DayTable) ([]string, error) {
-	_, users, err := b.storage.ListFiles(ctx, table.String(), true) // bypass cache for ease of testing
+func (b *BloomTSDBStore) UsersForPeriod(ctx context.Context, table config.DayTime) ([]string, error) {
+	_, users, err := b.storage.ListFiles(ctx, table.Table(), true) // bypass cache for ease of testing
 	return users, err
 }
 
-func (b *BloomTSDBStore) ResolveTSDBs(ctx context.Context, table DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) {
-	indices, err := b.storage.ListUserFiles(ctx, table.String(), tenant, true) // bypass cache for ease of testing
+func (b *BloomTSDBStore) ResolveTSDBs(ctx context.Context, table config.DayTime, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) {
+	indices, err := b.storage.ListUserFiles(ctx, table.Table(), tenant, true) // bypass cache for ease of testing
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to list user files")
 	}
@@ -80,14 +80,14 @@ func (b *BloomTSDBStore) ResolveTSDBs(ctx context.Context, table DayTable, tenan
 
 func (b *BloomTSDBStore) LoadTSDB(
 	ctx context.Context,
-	table DayTable,
+	table config.DayTime,
 	tenant string,
 	id tsdb.Identifier,
 	bounds v1.FingerprintBounds,
 ) (v1.CloseableIterator[*v1.Series], error) {
 	withCompression := id.Name() + gzipExtension
 
-	data, err := b.storage.GetUserFile(ctx, table.String(), tenant, withCompression)
+	data, err := b.storage.GetUserFile(ctx, table.Table(), tenant, withCompression)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get file")
 	}
@@ -244,11 +244,11 @@ func NewTSDBStores(
 	return res, nil
 }
 
-func (s *TSDBStores) storeForPeriod(table DayTable) (TSDBStore, error) {
+func (s *TSDBStores) storeForPeriod(table config.DayTime) (TSDBStore, error) {
 	for i := len(s.schemaCfg.Configs) - 1; i >= 0; i-- {
 		period := s.schemaCfg.Configs[i]
 
-		if !table.Before(DayTable(period.From.Time)) {
+		if !table.Before(period.From) {
 			// we have the desired period config
 
 			if s.stores[i] != nil {
@@ -260,19 +260,19 @@ func (s *TSDBStores) storeForPeriod(table DayTable) (TSDBStore, error) {
 			return nil, errors.Errorf(
 				"store for period is not of TSDB type (%s) while looking up store for (%v)",
 				period.IndexType,
-				table.ModelTime().Time(),
+				table,
 			)
 		}
 
 	}
 
 	return nil, fmt.Errorf(
-		"There is no store matching no matching period found for table (%v) -- too early",
-		table.ModelTime().Time(),
+		"there is no store matching no matching period found for table (%v) -- too early",
+		table,
 	)
 }
 
-func (s *TSDBStores) UsersForPeriod(ctx context.Context, table DayTable) ([]string, error) {
+func (s *TSDBStores) UsersForPeriod(ctx context.Context, table config.DayTime) ([]string, error) {
 	store, err := s.storeForPeriod(table)
 	if err != nil {
 		return nil, err
@@ -281,7 +281,7 @@ func (s *TSDBStores) UsersForPeriod(ctx context.Context, table DayTable) ([]stri
 	return store.UsersForPeriod(ctx, table)
 }
 
-func (s *TSDBStores) ResolveTSDBs(ctx context.Context, table DayTable, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) {
+func (s *TSDBStores) ResolveTSDBs(ctx context.Context, table config.DayTime, tenant string) ([]tsdb.SingleTenantTSDBIdentifier, error) {
 	store, err := s.storeForPeriod(table)
 	if err != nil {
 		return nil, err
@@ -292,7 +292,7 @@ func (s *TSDBStores) ResolveTSDBs(ctx context.Context, table DayTable, tenant st
 
 func (s *TSDBStores) LoadTSDB(
 	ctx context.Context,
-	table DayTable,
+	table config.DayTime,
 	tenant string,
 	id tsdb.Identifier,
 	bounds v1.FingerprintBounds,

pkg/bloomgateway/bloomgateway.go

@@ -369,7 +369,7 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk
 	tasksCh := make(chan Task, len(tasks))
 	for _, task := range tasks {
 		task := task
-		level.Info(logger).Log("msg", "enqueue task", "task", task.ID, "day", task.day, "series", len(task.series))
+		level.Info(logger).Log("msg", "enqueue task", "task", task.ID, "table", task.table, "series", len(task.series))
 		g.queue.Enqueue(tenantID, []string{}, task, func() {
 			// When enqueuing, we also add the task to the pending tasks
 			g.pendingTasks.Add(task.ID, task)

pkg/bloomgateway/cache.go

@@ -182,6 +182,7 @@ func NewBloomGatewayClientCacheMiddleware(
 		},
 		cacheGen,
 		retentionEnabled,
+		false,
 	)
 
 	return &ClientCache{