From 5d817121147711266cedbab067e41ff1abb64060 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Wed, 31 Jan 2024 13:15:08 +0100 Subject: [PATCH 1/2] Replace bloomshipper.Keyspace with v1.FingerprintBounds The latter struct has more utility functions to compare and operate on bounds. Signed-off-by: Christian Haudum --- pkg/bloomcompactor/bloomcompactor.go | 2 +- pkg/bloomgateway/processor.go | 8 ++-- pkg/bloomgateway/processor_test.go | 2 +- pkg/bloomgateway/util_test.go | 20 +++++----- pkg/bloomgateway/worker.go | 4 +- pkg/storage/bloom/v1/bounds.go | 5 +++ .../stores/shipper/bloomshipper/client.go | 6 ++- .../shipper/bloomshipper/client_test.go | 3 +- .../shipper/bloomshipper/fetcher_test.go | 23 ++++++----- .../stores/shipper/bloomshipper/shipper.go | 40 ++++++------------- .../shipper/bloomshipper/shipper_test.go | 27 +++++++------ 11 files changed, 67 insertions(+), 73 deletions(-) diff --git a/pkg/bloomcompactor/bloomcompactor.go b/pkg/bloomcompactor/bloomcompactor.go index 52799e498b51e..34885ae2d3947 100644 --- a/pkg/bloomcompactor/bloomcompactor.go +++ b/pkg/bloomcompactor/bloomcompactor.go @@ -503,7 +503,7 @@ func (c *Compactor) runCompact(ctx context.Context, logger log.Logger, job Job, } metaSearchParams := bloomshipper.MetaSearchParams{ TenantID: job.tenantID, - Keyspace: bloomshipper.Keyspace{Min: job.minFp, Max: job.maxFp}, + Keyspace: v1.NewBounds(job.minFp, job.maxFp), Interval: bloomshipper.Interval{Start: job.from, End: job.through}, } var metas []bloomshipper.Meta diff --git a/pkg/bloomgateway/processor.go b/pkg/bloomgateway/processor.go index 460ac3f44e038..5685851aef512 100644 --- a/pkg/bloomgateway/processor.go +++ b/pkg/bloomgateway/processor.go @@ -38,7 +38,7 @@ func (p *processor) run(ctx context.Context, tasks []Task) error { End: ts.Add(Day), } tenant := tasks[0].Tenant - err := p.processTasks(ctx, tenant, interval, []bloomshipper.Keyspace{{Min: 0, Max: math.MaxUint64}}, tasks) + err := p.processTasks(ctx, tenant, interval, []v1.FingerprintBounds{{Min: 0, Max: math.MaxUint64}}, tasks) if err != nil { for _, task := range tasks { task.CloseWithError(err) @@ -52,12 +52,12 @@ func (p *processor) run(ctx context.Context, tasks []Task) error { return nil } -func (p *processor) processTasks(ctx context.Context, tenant string, interval bloomshipper.Interval, keyspaces []bloomshipper.Keyspace, tasks []Task) error { +func (p *processor) processTasks(ctx context.Context, tenant string, interval bloomshipper.Interval, keyspaces []v1.FingerprintBounds, tasks []Task) error { minFpRange, maxFpRange := getFirstLast(keyspaces) metaSearch := bloomshipper.MetaSearchParams{ TenantID: tenant, Interval: interval, - Keyspace: bloomshipper.Keyspace{Min: minFpRange.Min, Max: maxFpRange.Max}, + Keyspace: v1.FingerprintBounds{Min: minFpRange.Min, Max: maxFpRange.Max}, } metas, err := p.store.FetchMetas(ctx, metaSearch) if err != nil { @@ -82,7 +82,7 @@ outer: for blockIter.Next() { bq := blockIter.At() for i, block := range data { - if block.blockRef.MinFingerprint == uint64(bq.MinFp) && block.blockRef.MaxFingerprint == uint64(bq.MaxFp) { + if block.blockRef.Bounds().Equal(bq.FingerprintBounds) { err := p.processBlock(ctx, bq.BlockQuerier, block.tasks) if err != nil { return err diff --git a/pkg/bloomgateway/processor_test.go b/pkg/bloomgateway/processor_test.go index 62c6d42ae18b3..6b43e688a4cc5 100644 --- a/pkg/bloomgateway/processor_test.go +++ b/pkg/bloomgateway/processor_test.go @@ -50,7 +50,7 @@ func (s *dummyStore) LoadBlocks(_ context.Context, refs []bloomshipper.BlockRef) for _, ref := range refs { for _, bq := range s.querieres { - if ref.MinFingerprint == uint64(bq.MinFp) && ref.MaxFingerprint == uint64(bq.MaxFp) { + if ref.Bounds().Equal(bq.FingerprintBounds) { result = append(result, bq) } } diff --git a/pkg/bloomgateway/util_test.go b/pkg/bloomgateway/util_test.go index 61825a8c677ae..969f0ddacd7b6 100644 --- a/pkg/bloomgateway/util_test.go +++ b/pkg/bloomgateway/util_test.go @@ -311,9 +311,8 @@ func createBlockQueriers(t *testing.T, numBlocks int, from, through model.Time, } blockQuerier, data := v1.MakeBlockQuerier(t, fromFp, throughFp, from, through) bq := bloomshipper.BlockQuerierWithFingerprintRange{ - BlockQuerier: blockQuerier, - MinFp: fromFp, - MaxFp: throughFp, + BlockQuerier: blockQuerier, + FingerprintBounds: v1.NewBounds(fromFp, throughFp), } bqs = append(bqs, bq) series = append(series, data) @@ -359,9 +358,8 @@ func createBlocks(t *testing.T, tenant string, n int, from, through model.Time, } blockQuerier, data := v1.MakeBlockQuerier(t, fromFp, throughFp, from, through) querier := bloomshipper.BlockQuerierWithFingerprintRange{ - BlockQuerier: blockQuerier, - MinFp: fromFp, - MaxFp: throughFp, + BlockQuerier: blockQuerier, + FingerprintBounds: v1.NewBounds(fromFp, throughFp), } queriers = append(queriers, querier) metas = append(metas, meta) @@ -392,8 +390,8 @@ func (s *mockBloomStore) GetBlockRefs(_ context.Context, tenant string, _ blooms for i := range s.bqs { blocks = append(blocks, bloomshipper.BlockRef{ Ref: bloomshipper.Ref{ - MinFingerprint: uint64(s.bqs[i].MinFp), - MaxFingerprint: uint64(s.bqs[i].MaxFp), + MinFingerprint: uint64(s.bqs[i].Min), + MaxFingerprint: uint64(s.bqs[i].Max), TenantID: tenant, }, }) @@ -421,7 +419,7 @@ func (s *mockBloomStore) Fetch(_ context.Context, _ string, _ []bloomshipper.Blo for _, bq := range shuffled { // ignore errors in the mock time.Sleep(s.delay) - err := callback(bq.BlockQuerier, uint64(bq.MinFp), uint64(bq.MaxFp)) + err := callback(bq.BlockQuerier, bq.FingerprintBounds) if err != nil { return err } @@ -459,8 +457,8 @@ func createBlockRefsFromBlockData(t *testing.T, tenant string, data []bloomshipp Ref: bloomshipper.Ref{ TenantID: tenant, TableName: "", - MinFingerprint: uint64(data[i].MinFp), - MaxFingerprint: uint64(data[i].MaxFp), + MinFingerprint: uint64(data[i].Min), + MaxFingerprint: uint64(data[i].Max), StartTimestamp: 0, EndTimestamp: 0, Checksum: 0, diff --git a/pkg/bloomgateway/worker.go b/pkg/bloomgateway/worker.go index 3b16fe4fdd7cf..5c6b8a76dbb22 100644 --- a/pkg/bloomgateway/worker.go +++ b/pkg/bloomgateway/worker.go @@ -244,9 +244,9 @@ func (w *worker) stopping(err error) error { } func (w *worker) processBlocksWithCallback(taskCtx context.Context, tenant string, blockRefs []bloomshipper.BlockRef, boundedRefs []boundedTasks) error { - return w.shipper.Fetch(taskCtx, tenant, blockRefs, func(bq *v1.BlockQuerier, minFp, maxFp uint64) error { + return w.shipper.Fetch(taskCtx, tenant, blockRefs, func(bq *v1.BlockQuerier, bounds v1.FingerprintBounds) error { for _, b := range boundedRefs { - if b.blockRef.MinFingerprint == minFp && b.blockRef.MaxFingerprint == maxFp { + if b.blockRef.Bounds().Equal(bounds) { return w.processBlock(bq, b.tasks) } } diff --git a/pkg/storage/bloom/v1/bounds.go b/pkg/storage/bloom/v1/bounds.go index fc22866285f82..8159501dd31d4 100644 --- a/pkg/storage/bloom/v1/bounds.go +++ b/pkg/storage/bloom/v1/bounds.go @@ -61,6 +61,11 @@ func (b FingerprintBounds) Within(target FingerprintBounds) bool { return b.Min >= target.Min && b.Max <= target.Max } +// Returns whether the fingerprint bounds is equal to the target bounds +func (b FingerprintBounds) Equal(target FingerprintBounds) bool { + return b.Min == target.Min && b.Max == target.Max +} + // Intersection returns the intersection of the two bounds func (b FingerprintBounds) Intersection(target FingerprintBounds) *FingerprintBounds { if !b.Overlaps(target) { diff --git a/pkg/storage/stores/shipper/bloomshipper/client.go b/pkg/storage/stores/shipper/bloomshipper/client.go index 8d980782d044f..835ee13686d37 100644 --- a/pkg/storage/stores/shipper/bloomshipper/client.go +++ b/pkg/storage/stores/shipper/bloomshipper/client.go @@ -51,6 +51,10 @@ type BlockRef struct { BlockPath string } +func (b *BlockRef) Bounds() v1.FingerprintBounds { + return v1.NewBounds(model.Fingerprint(b.MinFingerprint), model.Fingerprint(b.MaxFingerprint)) +} + type MetaRef struct { Ref FilePath string @@ -67,7 +71,7 @@ type Meta struct { type MetaSearchParams struct { TenantID string Interval Interval - Keyspace Keyspace + Keyspace v1.FingerprintBounds } type MetaClient interface { diff --git a/pkg/storage/stores/shipper/bloomshipper/client_test.go b/pkg/storage/stores/shipper/bloomshipper/client_test.go index 28e8c3a02f0e3..30aac5c901e08 100644 --- a/pkg/storage/stores/shipper/bloomshipper/client_test.go +++ b/pkg/storage/stores/shipper/bloomshipper/client_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/storage" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/config" ) @@ -72,7 +73,7 @@ func Test_BloomClient_FetchMetas(t *testing.T) { searchParams := MetaSearchParams{ TenantID: "tenantA", - Keyspace: Keyspace{Min: 50, Max: 150}, + Keyspace: v1.NewBounds(50, 150), Interval: Interval{Start: fixedDay.Add(-6 * day), End: fixedDay.Add(-1*day - 1*time.Hour)}, } diff --git a/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go b/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go index 3e0ea64c6f253..6359431eca9ad 100644 --- a/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go +++ b/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go @@ -12,12 +12,13 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" ) -func makeMetas(t *testing.T, schemaCfg config.SchemaConfig, ts model.Time, keyspaces []Keyspace) []Meta { +func makeMetas(t *testing.T, schemaCfg config.SchemaConfig, ts model.Time, keyspaces []v1.FingerprintBounds) []Meta { t.Helper() metas := make([]Meta, len(keyspaces)) @@ -76,23 +77,23 @@ func TestMetasFetcher(t *testing.T) { { name: "all metas found in cache", store: []Meta{}, - start: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), - end: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), - fetch: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), + start: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), }, { name: "no metas found in cache", - store: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), + store: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), start: []Meta{}, - end: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), - fetch: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), + end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), }, { name: "some metas found in cache", - store: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), - start: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}}), - end: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), - fetch: makeMetas(t, schemaCfg, now, []Keyspace{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), + store: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), + start: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), + fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), }, } diff --git a/pkg/storage/stores/shipper/bloomshipper/shipper.go b/pkg/storage/stores/shipper/bloomshipper/shipper.go index e3ab1db70c539..54c2185fae56d 100644 --- a/pkg/storage/stores/shipper/bloomshipper/shipper.go +++ b/pkg/storage/stores/shipper/bloomshipper/shipper.go @@ -32,25 +32,12 @@ func (i Interval) Cmp(other model.Time) v1.BoundsCheck { return v1.Overlap } -type Keyspace struct { - Min, Max model.Fingerprint -} - -func (r Keyspace) Cmp(other model.Fingerprint) v1.BoundsCheck { - if other < r.Min { - return v1.Before - } else if other > r.Max { - return v1.After - } - return v1.Overlap -} - type BlockQuerierWithFingerprintRange struct { *v1.BlockQuerier - MinFp, MaxFp model.Fingerprint + v1.FingerprintBounds } -type ForEachBlockCallback func(bq *v1.BlockQuerier, minFp, maxFp uint64) error +type ForEachBlockCallback func(bq *v1.BlockQuerier, bounds v1.FingerprintBounds) error type Interface interface { GetBlockRefs(ctx context.Context, tenant string, interval Interval) ([]BlockRef, error) @@ -93,8 +80,8 @@ func (s *Shipper) GetBlockRefs(ctx context.Context, tenantID string, interval In level.Debug(s.logger).Log("msg", "GetBlockRefs", "tenant", tenantID, "[", interval.Start, "", interval.End) // TODO(chaudum): The bloom gateway should not fetch blocks for the complete key space - keyspaces := []Keyspace{{0, math.MaxUint64}} - blockRefs, err := s.getActiveBlockRefs(ctx, tenantID, interval, keyspaces) + bounds := []v1.FingerprintBounds{v1.NewBounds(0, math.MaxUint64)} + blockRefs, err := s.getActiveBlockRefs(ctx, tenantID, interval, bounds) if err != nil { return nil, fmt.Errorf("error fetching active block references : %w", err) } @@ -136,7 +123,7 @@ func runCallback(callback ForEachBlockCallback, block blockWithQuerier) error { _ = b.Close() }(block) - err := callback(block.closableBlockQuerier.BlockQuerier, block.MinFingerprint, block.MaxFingerprint) + err := callback(block.closableBlockQuerier.BlockQuerier, block.Bounds()) if err != nil { return fmt.Errorf("error running callback function for block %s err: %w", block.BlockPath, err) } @@ -158,11 +145,11 @@ func getFirstLast[T any](s []T) (T, T) { return s[0], s[len(s)-1] } -func (s *Shipper) getActiveBlockRefs(ctx context.Context, tenantID string, interval Interval, keyspaces []Keyspace) ([]BlockRef, error) { - minFpRange, maxFpRange := getFirstLast(keyspaces) +func (s *Shipper) getActiveBlockRefs(ctx context.Context, tenantID string, interval Interval, bounds []v1.FingerprintBounds) ([]BlockRef, error) { + minFpRange, maxFpRange := getFirstLast(bounds) metas, err := s.store.FetchMetas(ctx, MetaSearchParams{ TenantID: tenantID, - Keyspace: Keyspace{Min: minFpRange.Min, Max: maxFpRange.Max}, + Keyspace: v1.NewBounds(minFpRange.Min, maxFpRange.Max), Interval: interval, }) if err != nil { @@ -170,10 +157,10 @@ func (s *Shipper) getActiveBlockRefs(ctx context.Context, tenantID string, inter } level.Debug(s.logger).Log("msg", "dowloaded metas", "count", len(metas)) - return BlocksForMetas(metas, interval, keyspaces), nil + return BlocksForMetas(metas, interval, bounds), nil } -func BlocksForMetas(metas []Meta, interval Interval, keyspaces []Keyspace) []BlockRef { +func BlocksForMetas(metas []Meta, interval Interval, keyspaces []v1.FingerprintBounds) []BlockRef { tombstones := make(map[string]interface{}) for _, meta := range metas { for _, tombstone := range meta.Tombstones { @@ -216,7 +203,7 @@ func BlocksForMetas(metas []Meta, interval Interval, keyspaces []Keyspace) []Blo // isOutsideRange tests if a given BlockRef b is outside of search boundaries // defined by min/max timestamp and min/max fingerprint. // Fingerprint ranges must be sorted in ascending order. -func isOutsideRange(b BlockRef, interval Interval, keyspaces []Keyspace) bool { +func isOutsideRange(b BlockRef, interval Interval, keyspaces []v1.FingerprintBounds) bool { // check time interval if interval.Cmp(b.EndTimestamp) == v1.Before || interval.Cmp(b.StartTimestamp) == v1.After { return true @@ -224,10 +211,7 @@ func isOutsideRange(b BlockRef, interval Interval, keyspaces []Keyspace) bool { // check fingerprint ranges for _, keyspace := range keyspaces { - if keyspace.Cmp(model.Fingerprint(b.MinFingerprint)) == v1.Before && keyspace.Cmp(model.Fingerprint(b.MaxFingerprint)) == v1.After { - return false - } - if keyspace.Cmp(model.Fingerprint(b.MinFingerprint)) == v1.Overlap || keyspace.Cmp(model.Fingerprint(b.MaxFingerprint)) == v1.Overlap { + if keyspace.Within(b.Bounds()) || keyspace.Overlaps(b.Bounds()) { return false } } diff --git a/pkg/storage/stores/shipper/bloomshipper/shipper_test.go b/pkg/storage/stores/shipper/bloomshipper/shipper_test.go index 57d360de3b80d..6b950ba1112ca 100644 --- a/pkg/storage/stores/shipper/bloomshipper/shipper_test.go +++ b/pkg/storage/stores/shipper/bloomshipper/shipper_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" ) @@ -48,7 +49,7 @@ func Test_Shipper_findBlocks(t *testing.T) { Start: ts.Add(-2 * time.Hour), End: ts.Add(-1 * time.Hour), } - blocks := BlocksForMetas(metas, interval, []Keyspace{{Min: 100, Max: 200}}) + blocks := BlocksForMetas(metas, interval, []v1.FingerprintBounds{{Min: 100, Max: 200}}) expectedBlockRefs := []BlockRef{ createMatchingBlockRef("block2"), @@ -101,7 +102,7 @@ func Test_Shipper_findBlocks(t *testing.T) { for name, data := range tests { t.Run(name, func(t *testing.T) { ref := createBlockRef("fake-block", data.minFingerprint, data.maxFingerprint, data.startTimestamp, data.endTimestamp) - blocks := BlocksForMetas([]Meta{{Blocks: []BlockRef{ref}}}, interval(300, 400), []Keyspace{{Min: 100, Max: 200}}) + blocks := BlocksForMetas([]Meta{{Blocks: []BlockRef{ref}}}, interval(300, 400), []v1.FingerprintBounds{{Min: 100, Max: 200}}) if data.filtered { require.Empty(t, blocks) return @@ -118,67 +119,67 @@ func TestIsOutsideRange(t *testing.T) { t.Run("is outside if startTs > through", func(t *testing.T) { b := createBlockRef("block", 0, math.MaxUint64, startTs, endTs) - isOutside := isOutsideRange(b, interval(0, 900), []Keyspace{}) + isOutside := isOutsideRange(b, interval(0, 900), []v1.FingerprintBounds{}) require.True(t, isOutside) }) t.Run("is outside if startTs == through ", func(t *testing.T) { b := createBlockRef("block", 0, math.MaxUint64, startTs, endTs) - isOutside := isOutsideRange(b, interval(900, 1000), []Keyspace{}) + isOutside := isOutsideRange(b, interval(900, 1000), []v1.FingerprintBounds{}) require.True(t, isOutside) }) t.Run("is outside if endTs < from", func(t *testing.T) { b := createBlockRef("block", 0, math.MaxUint64, startTs, endTs) - isOutside := isOutsideRange(b, interval(2100, 3000), []Keyspace{}) + isOutside := isOutsideRange(b, interval(2100, 3000), []v1.FingerprintBounds{}) require.True(t, isOutside) }) t.Run("is outside if endFp < first fingerprint", func(t *testing.T) { b := createBlockRef("block", 0, 90, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{100, 199}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{100, 199}}) require.True(t, isOutside) }) t.Run("is outside if startFp > last fingerprint", func(t *testing.T) { b := createBlockRef("block", 200, math.MaxUint64, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{0, 49}, {100, 149}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 49}, {100, 149}}) require.True(t, isOutside) }) t.Run("is outside if within gaps in fingerprints", func(t *testing.T) { b := createBlockRef("block", 100, 199, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) require.True(t, isOutside) }) t.Run("is not outside if within fingerprints 1", func(t *testing.T) { b := createBlockRef("block", 10, 90, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) require.False(t, isOutside) }) t.Run("is not outside if within fingerprints 2", func(t *testing.T) { b := createBlockRef("block", 210, 290, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) require.False(t, isOutside) }) t.Run("is not outside if spans across multiple fingerprint ranges", func(t *testing.T) { b := createBlockRef("block", 50, 250, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) require.False(t, isOutside) }) t.Run("is not outside if fingerprint range and time range are larger than block", func(t *testing.T) { b := createBlockRef("block", math.MaxUint64/3, math.MaxUint64/3*2, startTs, endTs) - isOutside := isOutsideRange(b, interval(0, 3000), []Keyspace{{0, math.MaxUint64}}) + isOutside := isOutsideRange(b, interval(0, 3000), []v1.FingerprintBounds{{0, math.MaxUint64}}) require.False(t, isOutside) }) t.Run("is not outside if block fingerprint range is bigger that search keyspace", func(t *testing.T) { b := createBlockRef("block", 0x0000, 0xffff, model.Earliest, model.Latest) - isOutside := isOutsideRange(b, interval(startTs, endTs), []Keyspace{{0x0100, 0xff00}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0x0100, 0xff00}}) require.False(t, isOutside) }) } From 1404477f6c56b6ac4c2b8317c938b32357660ff6 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Wed, 31 Jan 2024 14:08:18 +0100 Subject: [PATCH 2/2] Fix linter warnings Signed-off-by: Christian Haudum --- .../shipper/bloomshipper/fetcher_test.go | 20 +++++++++---------- .../shipper/bloomshipper/shipper_test.go | 19 +++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go b/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go index 6359431eca9ad..85117a718f629 100644 --- a/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go +++ b/pkg/storage/stores/shipper/bloomshipper/fetcher_test.go @@ -77,23 +77,23 @@ func TestMetasFetcher(t *testing.T) { { name: "all metas found in cache", store: []Meta{}, - start: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), - end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), - fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + start: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), + end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), + fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), }, { name: "no metas found in cache", - store: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + store: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), start: []Meta{}, - end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), - fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), + end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), + fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), }, { name: "some metas found in cache", - store: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), - start: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}}), - end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), - fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{0x0000, 0xffff}, {0x10000, 0x1ffff}}), + store: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}, {Min: 0x10000, Max: 0x1ffff}}), + start: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}}), + end: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}, {Min: 0x10000, Max: 0x1ffff}}), + fetch: makeMetas(t, schemaCfg, now, []v1.FingerprintBounds{{Min: 0x0000, Max: 0xffff}, {Min: 0x10000, Max: 0x1ffff}}), }, } diff --git a/pkg/storage/stores/shipper/bloomshipper/shipper_test.go b/pkg/storage/stores/shipper/bloomshipper/shipper_test.go index 6b950ba1112ca..d2311f808e26f 100644 --- a/pkg/storage/stores/shipper/bloomshipper/shipper_test.go +++ b/pkg/storage/stores/shipper/bloomshipper/shipper_test.go @@ -6,9 +6,10 @@ import ( "testing" "time" - v1 "github.com/grafana/loki/pkg/storage/bloom/v1" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" + + v1 "github.com/grafana/loki/pkg/storage/bloom/v1" ) func interval(start, end model.Time) Interval { @@ -137,49 +138,49 @@ func TestIsOutsideRange(t *testing.T) { t.Run("is outside if endFp < first fingerprint", func(t *testing.T) { b := createBlockRef("block", 0, 90, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{100, 199}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 100, Max: 199}}) require.True(t, isOutside) }) t.Run("is outside if startFp > last fingerprint", func(t *testing.T) { b := createBlockRef("block", 200, math.MaxUint64, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 49}, {100, 149}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 0, Max: 49}, {Min: 100, Max: 149}}) require.True(t, isOutside) }) t.Run("is outside if within gaps in fingerprints", func(t *testing.T) { b := createBlockRef("block", 100, 199, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 0, Max: 99}, {Min: 200, Max: 299}}) require.True(t, isOutside) }) t.Run("is not outside if within fingerprints 1", func(t *testing.T) { b := createBlockRef("block", 10, 90, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 0, Max: 99}, {Min: 200, Max: 299}}) require.False(t, isOutside) }) t.Run("is not outside if within fingerprints 2", func(t *testing.T) { b := createBlockRef("block", 210, 290, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 0, Max: 99}, {Min: 200, Max: 299}}) require.False(t, isOutside) }) t.Run("is not outside if spans across multiple fingerprint ranges", func(t *testing.T) { b := createBlockRef("block", 50, 250, startTs, endTs) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0, 99}, {200, 299}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 0, Max: 99}, {Min: 200, Max: 299}}) require.False(t, isOutside) }) t.Run("is not outside if fingerprint range and time range are larger than block", func(t *testing.T) { b := createBlockRef("block", math.MaxUint64/3, math.MaxUint64/3*2, startTs, endTs) - isOutside := isOutsideRange(b, interval(0, 3000), []v1.FingerprintBounds{{0, math.MaxUint64}}) + isOutside := isOutsideRange(b, interval(0, 3000), []v1.FingerprintBounds{{Min: 0, Max: math.MaxUint64}}) require.False(t, isOutside) }) t.Run("is not outside if block fingerprint range is bigger that search keyspace", func(t *testing.T) { b := createBlockRef("block", 0x0000, 0xffff, model.Earliest, model.Latest) - isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{0x0100, 0xff00}}) + isOutside := isOutsideRange(b, interval(startTs, endTs), []v1.FingerprintBounds{{Min: 0x0100, Max: 0xff00}}) require.False(t, isOutside) }) }