diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index d8b1a0dd5d466..e5bebfe044c80 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -22,7 +22,7 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" "github.com/grafana/loki/pkg/util/cfg" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/validation" diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index cbc5ef6b27317..d9cd00ac71652 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -41,7 +41,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/runtime" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" "github.com/grafana/loki/pkg/util" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/validation" diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index c3909ce69f651..5ee4e22bef696 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -4,7 +4,7 @@ import ( "time" "github.com/grafana/loki/pkg/distributor/shardstreams" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" ) // Limits is an interface for distributor limits/related configs diff --git a/pkg/ingester/index/bitprefix.go b/pkg/ingester/index/bitprefix.go index c65af7346734b..025005618d8c5 100644 --- a/pkg/ingester/index/bitprefix.go +++ b/pkg/ingester/index/bitprefix.go @@ -9,7 +9,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/querier/astmapper" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // BitPrefixInvertedIndex is another inverted index implementation diff --git a/pkg/ingester/index/bitprefix_test.go b/pkg/ingester/index/bitprefix_test.go index 8131db13e1f58..00640f42a23a7 100644 --- a/pkg/ingester/index/bitprefix_test.go +++ b/pkg/ingester/index/bitprefix_test.go @@ -11,7 +11,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/querier/astmapper" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func Test_BitPrefixGetShards(t *testing.T) { diff --git a/pkg/ingester/index/multi_test.go b/pkg/ingester/index/multi_test.go index 34fa3e6cf8d72..dc4568dab63b2 100644 --- a/pkg/ingester/index/multi_test.go +++ b/pkg/ingester/index/multi_test.go @@ -12,7 +12,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/querier/astmapper" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func MustParseDayTime(s string) config.DayTime { diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go index 769cb574c90ef..1627804b84c33 100644 --- a/pkg/logcli/query/query.go +++ b/pkg/logcli/query/query.go @@ -26,7 +26,7 @@ import ( "github.com/grafana/loki/pkg/storage" chunk "github.com/grafana/loki/pkg/storage/chunk/client" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" "github.com/grafana/loki/pkg/util/cfg" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/util/marshal" diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go index ac0b5d60e5ac6..814af40a46d0b 100644 --- a/pkg/logcli/query/query_test.go +++ b/pkg/logcli/query/query_test.go @@ -25,8 +25,8 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/shipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" "github.com/grafana/loki/pkg/util/marshal" ) @@ -518,7 +518,7 @@ func TestLoadFromURL(t *testing.T) { require.Error(t, err) require.Nil(t, client) - conf.StorageConfig.BoltDBShipperConfig = shipper.Config{ + conf.StorageConfig.BoltDBShipperConfig = boltdb.IndexCfg{ Config: indexshipper.Config{ SharedStoreType: config.StorageTypeFileSystem, }, diff --git a/pkg/logproto/extensions.go b/pkg/logproto/extensions.go index 66c5198f2ab5f..ee4e930941abb 100644 --- a/pkg/logproto/extensions.go +++ b/pkg/logproto/extensions.go @@ -9,7 +9,7 @@ import ( "github.com/dustin/go-humanize" "github.com/prometheus/common/model" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // This is the separator define in the Prometheus Labels.Hash function. diff --git a/pkg/loki/delete_store_listener.go b/pkg/loki/delete_store_listener.go index 846ce001ca176..327c949fbb4fb 100644 --- a/pkg/loki/delete_store_listener.go +++ b/pkg/loki/delete_store_listener.go @@ -3,7 +3,7 @@ package loki import ( "github.com/grafana/dskit/services" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" ) func deleteRequestsStoreListener(d deletion.DeleteRequestsClient) *listener { diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 3b914e62e709f..7c5c6a8f4a913 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -47,11 +47,11 @@ import ( internalserver "github.com/grafana/loki/pkg/server" "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - compactor_client "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/client" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + compactor_client "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/client" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" "github.com/grafana/loki/pkg/tracing" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/fakeauth" diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 55e47c2d84ed9..d4534016ff1a3 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -55,17 +55,17 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - compactor_client "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/client" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/client/grpc" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/generationnumber" "github.com/grafana/loki/pkg/storage/stores/series/index" - shipper_index "github.com/grafana/loki/pkg/storage/stores/shipper/index" - boltdb_shipper_compactor "github.com/grafana/loki/pkg/storage/stores/shipper/index/compactor" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" - "github.com/grafana/loki/pkg/storage/stores/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + boltdbcompactor "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + compactor_client "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/client" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/generationnumber" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" "github.com/grafana/loki/pkg/util/httpreq" "github.com/grafana/loki/pkg/util/limiter" util_log "github.com/grafana/loki/pkg/util/log" @@ -1168,7 +1168,7 @@ func (t *Loki) initCompactor() (services.Service, error) { return nil, err } - t.compactor.RegisterIndexCompactor(config.BoltDBShipperType, boltdb_shipper_compactor.NewIndexCompactor()) + t.compactor.RegisterIndexCompactor(config.BoltDBShipperType, boltdbcompactor.NewIndexCompactor()) t.compactor.RegisterIndexCompactor(config.TSDBType, tsdb.NewIndexCompactor()) t.Server.HTTP.Path("/compactor/ring").Methods("GET", "POST").Handler(t.compactor) @@ -1459,7 +1459,7 @@ func shipperQuerierIndexUpdateDelay(cacheValidity, resyncInterval time.Duration) // shipperIngesterIndexUploadDelay returns duration it could take for an index file containing id of a chunk to be uploaded to the shared store since it got flushed. func shipperIngesterIndexUploadDelay() time.Duration { - return shipper_index.ShardDBsByDuration + indexshipper.UploadInterval + return boltdb.ShardDBsByDuration + indexshipper.UploadInterval } // shipperMinIngesterQueryStoreDuration returns minimum duration(with some buffer) ingesters should query their stores to diff --git a/pkg/loki/modules_test.go b/pkg/loki/modules_test.go index 048c77a0d5ca0..4697dcf03e435 100644 --- a/pkg/loki/modules_test.go +++ b/pkg/loki/modules_test.go @@ -17,10 +17,9 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/shipper" - - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" ) func Test_calculateMaxLookBack(t *testing.T) { @@ -367,7 +366,7 @@ func minimalWorkingConfig(t *testing.T, dir, target string, cfgTransformers ...f // This would be overwritten by the default values setting. cfg.StorageConfig = storage.Config{ FSConfig: local.FSConfig{Directory: dir}, - BoltDBShipperConfig: shipper.Config{ + BoltDBShipperConfig: boltdb.IndexCfg{ Config: indexshipper.Config{ SharedStoreType: config.StorageTypeFileSystem, ActiveIndexDirectory: path.Join(dir, "index"), diff --git a/pkg/querier/astmapper/shard_summer.go b/pkg/querier/astmapper/shard_summer.go index df706fccbc56d..8226f35804eef 100644 --- a/pkg/querier/astmapper/shard_summer.go +++ b/pkg/querier/astmapper/shard_summer.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) const ( diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index df2664db5d484..6c7ab92470f3d 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -28,7 +28,7 @@ import ( "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/stores/index/stats" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" listutil "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/spanlogger" util_validation "github.com/grafana/loki/pkg/util/validation" diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index e64e9f559d077..79c706847cd37 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -22,7 +22,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/storage" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" "github.com/grafana/loki/pkg/validation" ) diff --git a/pkg/storage/config/schema_config.go b/pkg/storage/config/schema_config.go index 11a0948a589e2..5e1e94fc03e2b 100644 --- a/pkg/storage/config/schema_config.go +++ b/pkg/storage/config/schema_config.go @@ -19,7 +19,7 @@ import ( "github.com/grafana/loki/pkg/chunkenc" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index 3b6a89d4a115d..5611708df82b7 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -31,13 +31,13 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/testutils" "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/downloads" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/gatewayclient" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" - "github.com/grafana/loki/pkg/storage/stores/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/downloads" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/gatewayclient" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" "github.com/grafana/loki/pkg/util" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -45,7 +45,7 @@ import ( var ( indexGatewayClient index.Client // singleton for each period - boltdbIndexClientsWithShipper = make(map[config.DayTime]*shipper.IndexClient) + boltdbIndexClientsWithShipper = make(map[config.DayTime]*boltdb.IndexClient) supportedIndexTypes = []string{ config.BoltDBShipperType, @@ -100,7 +100,7 @@ func ResetBoltDBIndexClientsWithShipper() { client.Stop() } - boltdbIndexClientsWithShipper = make(map[config.DayTime]*shipper.IndexClient) + boltdbIndexClientsWithShipper = make(map[config.DayTime]*boltdb.IndexClient) if indexGatewayClient != nil { indexGatewayClient.Stop() @@ -332,9 +332,9 @@ type Config struct { DisableBroadIndexQueries bool `yaml:"disable_broad_index_queries"` MaxParallelGetChunk int `yaml:"max_parallel_get_chunk"` - MaxChunkBatchSize int `yaml:"max_chunk_batch_size"` - BoltDBShipperConfig shipper.Config `yaml:"boltdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in the form of boltdb files. Required fields only required when boltdb-shipper is defined in config."` - TSDBShipperConfig tsdb.IndexCfg `yaml:"tsdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in a prometheus TSDB-like format. Required fields only required when TSDB is defined in config."` + MaxChunkBatchSize int `yaml:"max_chunk_batch_size"` + BoltDBShipperConfig boltdb.IndexCfg `yaml:"boltdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in the form of boltdb files. Required fields only required when boltdb-shipper is defined in config."` + TSDBShipperConfig tsdb.IndexCfg `yaml:"tsdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in a prometheus TSDB-like format. Required fields only required when TSDB is defined in config."` // Config for using AsyncStore when using async index stores like `boltdb-shipper`. // It is required for getting chunk ids of recently flushed chunks from the ingesters. @@ -440,7 +440,7 @@ func NewIndexClient(periodCfg config.PeriodConfig, tableRange config.TableRange, if shardingStrategy != nil { filterFn = shardingStrategy.FilterTenants } - indexClient, err := shipper.NewIndexClient(cfg.BoltDBShipperConfig, objectClient, limits, filterFn, tableRange, registerer, logger) + indexClient, err := boltdb.NewIndexClient(cfg.BoltDBShipperConfig, objectClient, limits, filterFn, tableRange, registerer, logger) if err != nil { return nil, err } diff --git a/pkg/storage/factory_test.go b/pkg/storage/factory_test.go index 8a8c7dcb4d12f..faeb6f038617d 100644 --- a/pkg/storage/factory_test.go +++ b/pkg/storage/factory_test.go @@ -15,8 +15,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/cassandra" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/shipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/validation" ) @@ -94,7 +94,7 @@ func TestNamedStores(t *testing.T) { tempDir := t.TempDir() // config for BoltDB Shipper - boltdbShipperConfig := shipper.Config{} + boltdbShipperConfig := boltdb.IndexCfg{} flagext.DefaultValues(&boltdbShipperConfig) boltdbShipperConfig.ActiveIndexDirectory = path.Join(tempDir, "index") boltdbShipperConfig.SharedStoreType = "named-store" diff --git a/pkg/storage/store.go b/pkg/storage/store.go index 5cca9deb62048..525c1535726de 100644 --- a/pkg/storage/store.go +++ b/pkg/storage/store.go @@ -28,12 +28,12 @@ import ( "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores" "github.com/grafana/loki/pkg/storage/stores/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/gatewayclient" "github.com/grafana/loki/pkg/storage/stores/series" series_index "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" - "github.com/grafana/loki/pkg/storage/stores/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/gatewayclient" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/deletion" ) diff --git a/pkg/storage/store_test.go b/pkg/storage/store_test.go index eb9ea6306cc8f..6eef4433ab577 100644 --- a/pkg/storage/store_test.go +++ b/pkg/storage/store_test.go @@ -29,9 +29,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/shipper" - "github.com/grafana/loki/pkg/storage/stores/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/util/marshal" "github.com/grafana/loki/pkg/validation" @@ -1189,11 +1189,9 @@ func TestStore_MultiPeriod(t *testing.T) { shipperConfig.Mode = indexshipper.ModeReadWrite cfg := Config{ - FSConfig: local.FSConfig{Directory: path.Join(tempDir, "chunks")}, - BoltDBShipperConfig: shipper.Config{ - Config: shipperConfig, - }, - TSDBShipperConfig: tsdb.IndexCfg{Config: shipperConfig, CachePostings: false}, + FSConfig: local.FSConfig{Directory: path.Join(tempDir, "chunks")}, + BoltDBShipperConfig: boltdb.IndexCfg{Config: shipperConfig}, + TSDBShipperConfig: tsdb.IndexCfg{Config: shipperConfig, CachePostings: false}, NamedStores: NamedStores{ Filesystem: map[string]NamedFSConfig{ "named-store": {Directory: path.Join(tempDir, "named-store")}, @@ -1507,7 +1505,7 @@ func TestStore_BoltdbTsdbSameIndexPrefix(t *testing.T) { require.NoError(t, err) // config for BoltDB Shipper - boltdbShipperConfig := shipper.Config{} + boltdbShipperConfig := boltdb.IndexCfg{} flagext.DefaultValues(&boltdbShipperConfig) boltdbShipperConfig.ActiveIndexDirectory = path.Join(tempDir, "index") boltdbShipperConfig.SharedStoreType = config.StorageTypeFileSystem @@ -1516,7 +1514,7 @@ func TestStore_BoltdbTsdbSameIndexPrefix(t *testing.T) { boltdbShipperConfig.IngesterName = ingesterName // config for tsdb Shipper - tsdbShipperConfig := indexshipper.Config{} + tsdbShipperConfig := tsdb.IndexCfg{} flagext.DefaultValues(&tsdbShipperConfig) tsdbShipperConfig.ActiveIndexDirectory = path.Join(tempDir, "tsdb-index") tsdbShipperConfig.SharedStoreType = config.StorageTypeFileSystem @@ -1531,7 +1529,7 @@ func TestStore_BoltdbTsdbSameIndexPrefix(t *testing.T) { cfg := Config{ FSConfig: local.FSConfig{Directory: path.Join(tempDir, "chunks")}, BoltDBShipperConfig: boltdbShipperConfig, - TSDBShipperConfig: tsdb.IndexCfg{Config: tsdbShipperConfig}, + TSDBShipperConfig: tsdbShipperConfig, } schemaConfig := config.SchemaConfig{ diff --git a/pkg/storage/stores/index/stats/stats.go b/pkg/storage/stores/index/stats/stats.go index 1a837e502076d..82d0791c1dc7c 100644 --- a/pkg/storage/stores/index/stats/stats.go +++ b/pkg/storage/stores/index/stats/stats.go @@ -8,7 +8,7 @@ import ( "github.com/willf/bloom" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) var BloomPool PoolBloom diff --git a/pkg/storage/stores/series/index/index.go b/pkg/storage/stores/series/index/index.go index c83d8821668e3..5011c7e8f0815 100644 --- a/pkg/storage/stores/series/index/index.go +++ b/pkg/storage/stores/series/index/index.go @@ -75,3 +75,21 @@ type Entry struct { // New for v6 schema, label value is not written as part of the range key. Value []byte } + +func QueryKey(q Query) string { + ret := q.TableName + sep + q.HashValue + + if len(q.RangeValuePrefix) != 0 { + ret += sep + string(q.RangeValuePrefix) + } + + if len(q.RangeValueStart) != 0 { + ret += sep + string(q.RangeValueStart) + } + + if len(q.ValueEqual) != 0 { + ret += sep + string(q.ValueEqual) + } + + return ret +} diff --git a/pkg/storage/stores/shipper/index/metrics.go b/pkg/storage/stores/shipper/index/metrics.go deleted file mode 100644 index d8fd5655c3bb5..0000000000000 --- a/pkg/storage/stores/shipper/index/metrics.go +++ /dev/null @@ -1,20 +0,0 @@ -package index - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -type metrics struct { - openExistingFileFailuresTotal prometheus.Counter -} - -func newMetrics(r prometheus.Registerer) *metrics { - return &metrics{ - openExistingFileFailuresTotal: promauto.With(r).NewCounter(prometheus.CounterOpts{ - Namespace: "loki_boltdb_shipper", - Name: "open_existing_file_failures_total", - Help: "Total number of failures in opening of existing files while loading active index tables during startup", - }), - } -} diff --git a/pkg/storage/stores/shipper/index/compactor/compacted_index.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/compacted_index.go similarity index 89% rename from pkg/storage/stores/shipper/index/compactor/compacted_index.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/compacted_index.go index 557b49fbc2b6e..acdd7ab655334 100644 --- a/pkg/storage/stores/shipper/index/compactor/compacted_index.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/compacted_index.go @@ -15,11 +15,11 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - shipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - series_index "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + seriesindex "github.com/grafana/loki/pkg/storage/stores/series/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + shipperutil "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" ) type CompactedIndex struct { @@ -165,7 +165,7 @@ func (c *CompactedIndex) CleanupSeries(userID []byte, lbls labels.Labels) error return c.seriesCleaner.CleanupSeries(userID, lbls) } -func (c *CompactedIndex) ToIndexFile() (shipper_index.Index, error) { +func (c *CompactedIndex) ToIndexFile() (shipperindex.Index, error) { if c.boltdbTx != nil { err := c.boltdbTx.Commit() if err != nil { @@ -178,9 +178,9 @@ func (c *CompactedIndex) ToIndexFile() (shipper_index.Index, error) { if c.compactedFileRecreated { fileNameFormat = "%s" + recreatedCompactedDBSuffix } - fileName := fmt.Sprintf(fileNameFormat, shipper_util.BuildIndexFileName(c.tableName, uploaderName, fmt.Sprint(time.Now().Unix()))) + fileName := fmt.Sprintf(fileNameFormat, shipperutil.BuildIndexFileName(c.tableName, uploaderName, fmt.Sprint(time.Now().Unix()))) - idxFile := indexfile.BoltDBToIndexFile(c.compactedFile, fileName) + idxFile := boltdb.BoltDBToIndexFile(c.compactedFile, fileName) c.compactedFile = nil return idxFile, nil } @@ -211,11 +211,11 @@ type chunkIndexer struct { scfg config.SchemaConfig tableName string - seriesStoreSchema series_index.SeriesStoreSchema + seriesStoreSchema seriesindex.SeriesStoreSchema } func newChunkIndexer(bucket *bbolt.Bucket, periodConfig config.PeriodConfig, tableName string) (*chunkIndexer, error) { - seriesStoreSchema, err := series_index.CreateSchema(periodConfig) + seriesStoreSchema, err := seriesindex.CreateSchema(periodConfig) if err != nil { return nil, err } diff --git a/pkg/storage/stores/shipper/index/compactor/compacted_index_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/compacted_index_test.go similarity index 94% rename from pkg/storage/stores/shipper/index/compactor/compacted_index_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/compacted_index_test.go index e662c48649415..0a3468812606b 100644 --- a/pkg/storage/stores/shipper/index/compactor/compacted_index_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/compacted_index_test.go @@ -16,8 +16,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -71,7 +71,7 @@ func TestCompactedIndex_IndexProcessor(t *testing.T) { require.NoError(t, os.Remove(path)) }() - modifiedBoltDB := indexFile.(*indexfile.IndexFile).GetBoltDB() + modifiedBoltDB := indexFile.(*boltdb.IndexFile).GetBoltDB() err = modifiedBoltDB.View(func(tx *bbolt.Tx) error { return tx.Bucket(local.IndexBucketName).ForEach(func(k, _ []byte) error { diff --git a/pkg/storage/stores/shipper/index/compactor/index.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index.go similarity index 98% rename from pkg/storage/stores/shipper/index/compactor/index.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index.go index 724acea579c41..edfdb151176c0 100644 --- a/pkg/storage/stores/shipper/index/compactor/index.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/common/model" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" ) const ( diff --git a/pkg/storage/stores/shipper/index/compactor/index_compactor.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index_compactor.go similarity index 93% rename from pkg/storage/stores/shipper/index/compactor/index_compactor.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index_compactor.go index 02280655a86b2..1f4a1c7d91a21 100644 --- a/pkg/storage/stores/shipper/index/compactor/index_compactor.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index_compactor.go @@ -6,7 +6,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" ) const ( diff --git a/pkg/storage/stores/shipper/index/compactor/index_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index_test.go similarity index 100% rename from pkg/storage/stores/shipper/index/compactor/index_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/index_test.go diff --git a/pkg/storage/stores/shipper/index/compactor/iterator.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/iterator.go similarity index 97% rename from pkg/storage/stores/shipper/index/compactor/iterator.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/iterator.go index a09bf4b25f24e..0a192398aabe9 100644 --- a/pkg/storage/stores/shipper/index/compactor/iterator.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/iterator.go @@ -9,8 +9,8 @@ import ( "go.etcd.io/bbolt" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" series_index "github.com/grafana/loki/pkg/storage/stores/series/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" ) const ( diff --git a/pkg/storage/stores/shipper/index/compactor/iterator_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/iterator_test.go similarity index 98% rename from pkg/storage/stores/shipper/index/compactor/iterator_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/iterator_test.go index 4767429d701ba..adeb643a99ea8 100644 --- a/pkg/storage/stores/shipper/index/compactor/iterator_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/iterator_test.go @@ -19,7 +19,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" ) func Test_ChunkIterator(t *testing.T) { diff --git a/pkg/storage/stores/shipper/index/compactor/pool.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/pool.go similarity index 100% rename from pkg/storage/stores/shipper/index/compactor/pool.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/pool.go diff --git a/pkg/storage/stores/shipper/index/compactor/series.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/series.go similarity index 100% rename from pkg/storage/stores/shipper/index/compactor/series.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/series.go diff --git a/pkg/storage/stores/shipper/index/compactor/table_compactor.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/table_compactor.go similarity index 99% rename from pkg/storage/stores/shipper/index/compactor/table_compactor.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/table_compactor.go index 6f6c20daf0faf..65cedc82116f4 100644 --- a/pkg/storage/stores/shipper/index/compactor/table_compactor.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/table_compactor.go @@ -17,9 +17,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/pkg/storage/stores/shipper/index/compactor/table_compactor_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/table_compactor_test.go similarity index 98% rename from pkg/storage/stores/shipper/index/compactor/table_compactor_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/table_compactor_test.go index 0d1351568170f..95ea695d104e7 100644 --- a/pkg/storage/stores/shipper/index/compactor/table_compactor_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/table_compactor_test.go @@ -20,9 +20,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/shipper/testutil" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/testutil" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/shipper/index/compactor/util.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util.go similarity index 100% rename from pkg/storage/stores/shipper/index/compactor/util.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util.go diff --git a/pkg/storage/stores/shipper/index/compactor/util_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go similarity index 97% rename from pkg/storage/stores/shipper/index/compactor/util_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go index f8e83595e61c8..d5daa426ed6d8 100644 --- a/pkg/storage/stores/shipper/index/compactor/util_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/compactor/util_test.go @@ -21,9 +21,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/shipper" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb" + shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/validation" ) @@ -224,7 +224,7 @@ func newTestStore(t testing.TB, clientMetrics storage.ClientMetrics) *testStore }, MaxParallelGetChunk: 150, - BoltDBShipperConfig: shipper.Config{ + BoltDBShipperConfig: boltdb.IndexCfg{ Config: indexshipper.Config{ ActiveIndexDirectory: indexDir, SharedStoreType: "filesystem", diff --git a/pkg/storage/stores/shipper/index/indexfile/indexfile.go b/pkg/storage/stores/shipper/indexshipper/boltdb/index.go similarity index 93% rename from pkg/storage/stores/shipper/index/indexfile/indexfile.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/index.go index 1ea2c6ba3f794..6d548f02c3348 100644 --- a/pkg/storage/stores/shipper/index/indexfile/indexfile.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/index.go @@ -1,4 +1,4 @@ -package indexfile +package boltdb import ( "context" @@ -14,9 +14,9 @@ import ( "go.etcd.io/bbolt" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" series_index "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -44,6 +44,7 @@ func OpenIndexFile(path string) (index.Index, error) { }, nil } +// nolint:revive func BoltDBToIndexFile(boltdbFile *bbolt.DB, name string) index.Index { return &IndexFile{ boltDB: boltdbFile, diff --git a/pkg/storage/stores/shipper/shipper_index_client.go b/pkg/storage/stores/shipper/indexshipper/boltdb/index_client.go similarity index 65% rename from pkg/storage/stores/shipper/shipper_index_client.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/index_client.go index aea57985dd3f1..20a2235353f40 100644 --- a/pkg/storage/stores/shipper/shipper_index_client.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/index_client.go @@ -1,4 +1,4 @@ -package shipper +package boltdb import ( "context" @@ -10,61 +10,76 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/instrument" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "go.etcd.io/bbolt" "github.com/grafana/loki/pkg/storage/chunk/client" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/downloads" series_index "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/downloads" ) -type Config struct { +type indexClientMetrics struct { + // duration in seconds spent in serving request on index managed by BoltDB Shipper + requestDurationSeconds *prometheus.HistogramVec +} + +func newIndexClientMetrics(r prometheus.Registerer) *indexClientMetrics { + return &indexClientMetrics{ + requestDurationSeconds: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "loki_boltdb_shipper", + Name: "request_duration_seconds", + Help: "Time (in seconds) spent serving requests when using boltdb shipper", + Buckets: instrument.DefBuckets, + }, []string{"operation", "status_code"}), + } +} + +type IndexCfg struct { indexshipper.Config `yaml:",inline"` BuildPerTenantIndex bool `yaml:"build_per_tenant_index"` } // RegisterFlags registers flags. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { +func (cfg *IndexCfg) RegisterFlags(f *flag.FlagSet) { cfg.RegisterFlagsWithPrefix("boltdb.", f) } -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { +func (cfg *IndexCfg) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { cfg.Config.RegisterFlagsWithPrefix(prefix, f) f.BoolVar(&cfg.BuildPerTenantIndex, prefix+"shipper.build-per-tenant-index", false, "Build per tenant index files") } -func (cfg *Config) Validate() error { +func (cfg *IndexCfg) Validate() error { return cfg.Config.Validate() } type writer interface { - ForEach(ctx context.Context, tableName string, callback func(boltdb *bbolt.DB) error) error + ForEach(ctx context.Context, tableName string, callback func(b *bbolt.DB) error) error BatchWrite(ctx context.Context, batch series_index.WriteBatch) error Stop() } type IndexClient struct { - cfg Config + cfg IndexCfg indexShipper indexshipper.IndexShipper writer writer - querier index.Querier + querier Querier - metrics *metrics + metrics *indexClientMetrics logger log.Logger stopOnce sync.Once } -// NewIndexClient creates a shipper for syncing local objects with a store -func NewIndexClient(cfg Config, storageClient client.ObjectClient, limits downloads.Limits, +// New creates a shipper for syncing local objects with a store +func NewIndexClient(cfg IndexCfg, storageClient client.ObjectClient, limits downloads.Limits, tenantFilter downloads.TenantFilter, tableRange config.TableRange, registerer prometheus.Registerer, logger log.Logger) (*IndexClient, error) { i := IndexClient{ cfg: cfg, - metrics: newMetrics(registerer), + metrics: newIndexClientMetrics(registerer), logger: logger, } @@ -82,7 +97,7 @@ func (i *IndexClient) init(storageClient client.ObjectClient, limits downloads.L tenantFilter downloads.TenantFilter, tableRange config.TableRange, registerer prometheus.Registerer) error { var err error i.indexShipper, err = indexshipper.NewIndexShipper(i.cfg.Config, storageClient, limits, tenantFilter, - indexfile.OpenIndexFile, tableRange, prometheus.WrapRegistererWithPrefix("loki_boltdb_shipper_", registerer), i.logger) + OpenIndexFile, tableRange, prometheus.WrapRegistererWithPrefix("loki_boltdb_shipper_", registerer), i.logger) if err != nil { return err } @@ -93,19 +108,19 @@ func (i *IndexClient) init(storageClient client.ObjectClient, limits downloads.L return err } - cfg := index.Config{ + cfg := Config{ Uploader: uploader, IndexDir: i.cfg.ActiveIndexDirectory, DBRetainPeriod: i.cfg.IngesterDBRetainPeriod, MakePerTenantBuckets: i.cfg.BuildPerTenantIndex, } - i.writer, err = index.NewTableManager(cfg, i.indexShipper, tableRange, registerer, i.logger) + i.writer, err = NewTableManager(cfg, i.indexShipper, tableRange, registerer, i.logger) if err != nil { return err } } - i.querier = index.NewQuerier(i.writer, i.indexShipper) + i.querier = NewQuerier(i.writer, i.indexShipper) return nil } diff --git a/pkg/storage/stores/shipper/index/querier.go b/pkg/storage/stores/shipper/indexshipper/boltdb/querier.go similarity index 68% rename from pkg/storage/stores/shipper/index/querier.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/querier.go index 9f60388ea8d70..54d6034b36be1 100644 --- a/pkg/storage/stores/shipper/index/querier.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/querier.go @@ -1,4 +1,4 @@ -package index +package boltdb import ( "context" @@ -7,14 +7,13 @@ import ( "github.com/grafana/dskit/tenant" "go.etcd.io/bbolt" - shipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" - "github.com/grafana/loki/pkg/storage/stores/shipper/util" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" ) type Writer interface { - ForEach(ctx context.Context, tableName string, callback func(boltdb *bbolt.DB) error) error + ForEach(ctx context.Context, tableName string, callback func(b *bbolt.DB) error) error } type Querier interface { @@ -46,21 +45,21 @@ func (q *querier) QueryPages(ctx context.Context, queries []index.Query, callbac err := util.DoParallelQueries(ctx, func(ctx context.Context, queries []index.Query, callback index.QueryPagesCallback) error { // writer could be nil when running in ReadOnly mode if q.writer != nil { - err := q.writer.ForEach(ctx, table, func(boltdb *bbolt.DB) error { - return indexfile.QueryBoltDB(ctx, boltdb, userIDBytes, queries, callback) + err := q.writer.ForEach(ctx, table, func(b *bbolt.DB) error { + return QueryBoltDB(ctx, b, userIDBytes, queries, callback) }) if err != nil { return err } } - return q.indexShipper.ForEach(ctx, table, userID, func(_ bool, idx shipper_index.Index) error { - boltdbIndexFile, ok := idx.(*indexfile.IndexFile) + return q.indexShipper.ForEach(ctx, table, userID, func(_ bool, idx shipperindex.Index) error { + boltdbIndexFile, ok := idx.(*IndexFile) if !ok { return fmt.Errorf("unexpected index type %T", idx) } - return indexfile.QueryBoltDB(ctx, boltdbIndexFile.GetBoltDB(), userIDBytes, queries, callback) + return QueryBoltDB(ctx, boltdbIndexFile.GetBoltDB(), userIDBytes, queries, callback) }) }, queries, callback) if err != nil { diff --git a/pkg/storage/stores/shipper/index/table.go b/pkg/storage/stores/shipper/indexshipper/boltdb/table.go similarity index 95% rename from pkg/storage/stores/shipper/index/table.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/table.go index 10cde71017ba4..0db3ffa7f0a68 100644 --- a/pkg/storage/stores/shipper/index/table.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/table.go @@ -1,4 +1,4 @@ -package index +package boltdb import ( "context" @@ -18,8 +18,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -31,6 +30,7 @@ const ( snapshotFileSuffix = ".snapshot" ) +// nolint:revive type BoltDBIndexClient interface { QueryWithCursor(_ context.Context, c *bbolt.Cursor, query index.Query, callback index.QueryPagesCallback) error WriteToDB(ctx context.Context, db *bbolt.DB, bucketName []byte, writes local.TableWrites) error @@ -71,7 +71,7 @@ func NewTable(path, uploader string, indexShipper Shipper, makePerTenantBuckets } // LoadTable loads local dbs belonging to the table and creates a new Table with references to dbs if there are any otherwise it doesn't create a table -func LoadTable(path, uploader string, indexShipper Shipper, makePerTenantBuckets bool, metrics *metrics) (*Table, error) { +func LoadTable(path, uploader string, indexShipper Shipper, makePerTenantBuckets bool, metrics *tableManagerMetrics) (*Table, error) { dbs, err := loadBoltDBsFromDir(path, metrics) if err != nil { return nil, err @@ -172,7 +172,7 @@ func (lt *Table) Snapshot() error { return nil } -func (lt *Table) ForEach(_ context.Context, callback func(boltdb *bbolt.DB) error) error { +func (lt *Table) ForEach(_ context.Context, callback func(b *bbolt.DB) error) error { lt.dbSnapshotsMtx.RLock() defer lt.dbSnapshotsMtx.RUnlock() @@ -324,7 +324,7 @@ func (lt *Table) handoverIndexesToShipper(force bool) ([]string, error) { continue } - err = lt.indexShipper.AddIndex(lt.name, "", indexfile.BoltDBToIndexFile(db, lt.buildFileName(name))) + err = lt.indexShipper.AddIndex(lt.name, "", BoltDBToIndexFile(db, lt.buildFileName(name))) if err != nil { return nil, err } @@ -348,7 +348,7 @@ func (lt *Table) buildFileName(dbName string) string { return fileName } -func loadBoltDBsFromDir(dir string, metrics *metrics) (map[string]*bbolt.DB, error) { +func loadBoltDBsFromDir(dir string, metrics *tableManagerMetrics) (map[string]*bbolt.DB, error) { dbs := map[string]*bbolt.DB{} dirEntries, err := os.ReadDir(dir) if err != nil { @@ -361,7 +361,7 @@ func loadBoltDBsFromDir(dir string, metrics *metrics) (map[string]*bbolt.DB, err } fullPath := filepath.Join(dir, entry.Name()) - if strings.HasSuffix(entry.Name(), indexfile.TempFileSuffix) || strings.HasSuffix(entry.Name(), snapshotFileSuffix) { + if strings.HasSuffix(entry.Name(), TempFileSuffix) || strings.HasSuffix(entry.Name(), snapshotFileSuffix) { // If an ingester is killed abruptly in the middle of an upload operation it could leave out a temp file which holds the snapshot of db for uploading. // Cleaning up those temp files to avoid problems. if err := os.Remove(fullPath); err != nil { diff --git a/pkg/storage/stores/shipper/index/table_manager.go b/pkg/storage/stores/shipper/indexshipper/boltdb/table_manager.go similarity index 88% rename from pkg/storage/stores/shipper/index/table_manager.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/table_manager.go index edfd8c497121a..29fa04deb326b 100644 --- a/pkg/storage/stores/shipper/index/table_manager.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/table_manager.go @@ -1,4 +1,4 @@ -package index +package boltdb import ( "context" @@ -13,16 +13,31 @@ import ( "github.com/go-kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "go.etcd.io/bbolt" "github.com/grafana/loki/pkg/storage/chunk/client/local" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - shipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" "github.com/grafana/loki/pkg/storage/stores/series/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" ) +type tableManagerMetrics struct { + openExistingFileFailuresTotal prometheus.Counter +} + +func newTableManagerMetrics(r prometheus.Registerer) *tableManagerMetrics { + return &tableManagerMetrics{ + openExistingFileFailuresTotal: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: "loki_boltdb_shipper", + Name: "open_existing_file_failures_total", + Help: "Total number of failures in opening of existing files while loading active index tables during startup", + }), + } +} + type Config struct { Uploader string IndexDir string @@ -34,7 +49,7 @@ type TableManager struct { cfg Config indexShipper Shipper - metrics *metrics + metrics *tableManagerMetrics logger log.Logger tables map[string]*Table tableRange config.TableRange @@ -46,8 +61,8 @@ type TableManager struct { } type Shipper interface { - AddIndex(tableName, userID string, index shipper_index.Index) error - ForEach(ctx context.Context, tableName, userID string, callback shipper_index.ForEachIndexCallback) error + AddIndex(tableName, userID string, index shipperindex.Index) error + ForEach(ctx context.Context, tableName, userID string, callback shipperindex.ForEachIndexCallback) error } func NewTableManager(cfg Config, indexShipper Shipper, tableRange config.TableRange, registerer prometheus.Registerer, logger log.Logger) (*TableManager, error) { @@ -60,7 +75,7 @@ func NewTableManager(cfg Config, indexShipper Shipper, tableRange config.TableRa tm := TableManager{ cfg: cfg, indexShipper: indexShipper, - metrics: newMetrics(registerer), + metrics: newTableManagerMetrics(registerer), tableRange: tableRange, ctx: ctx, cancel: cancel, diff --git a/pkg/storage/stores/shipper/index/table_manager_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/table_manager_test.go similarity index 89% rename from pkg/storage/stores/shipper/index/table_manager_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/table_manager_test.go index b9d04cd0d083f..fc175e0e2c6ce 100644 --- a/pkg/storage/stores/shipper/index/table_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/table_manager_test.go @@ -1,4 +1,4 @@ -package index +package boltdb import ( "context" @@ -15,10 +15,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - index_shipper "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" - "github.com/grafana/loki/pkg/storage/stores/shipper/testutil" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/testutil" ) const indexTablePeriod = 24 * time.Hour @@ -142,16 +141,16 @@ func TestLoadTables(t *testing.T) { for tableName, expectedIndex := range expectedTables { // loaded tables should not have any index files, it should have handed them over to index shipper testutil.VerifyIndexes(t, userID, []index.Query{{TableName: tableName}}, - func(ctx context.Context, table string, callback func(boltdb *bbolt.DB) error) error { + func(ctx context.Context, table string, callback func(b *bbolt.DB) error) error { return tm.tables[tableName].ForEach(ctx, callback) }, 0, 0) // see if index shipper has the index files testutil.VerifyIndexes(t, userID, []index.Query{{TableName: tableName}}, - func(ctx context.Context, table string, callback func(boltdb *bbolt.DB) error) error { - return tm.indexShipper.ForEach(ctx, table, userID, func(_ bool, index index_shipper.Index) error { - return callback(index.(*indexfile.IndexFile).GetBoltDB()) + func(ctx context.Context, table string, callback func(b *bbolt.DB) error) error { + return tm.indexShipper.ForEach(ctx, table, userID, func(_ bool, index shipperindex.Index) error { + return callback(index.(*IndexFile).GetBoltDB()) }) }, expectedIndex.start, expectedIndex.numRecords) @@ -186,7 +185,7 @@ func TestTableManager_BatchWrite(t *testing.T) { for tableName, expectedIndex := range tc { require.NoError(t, tm.tables[tableName].Snapshot()) testutil.VerifyIndexes(t, userID, []index.Query{{TableName: tableName}}, - func(ctx context.Context, table string, callback func(boltdb *bbolt.DB) error) error { + func(ctx context.Context, table string, callback func(b *bbolt.DB) error) error { return tm.tables[tableName].ForEach(context.Background(), callback) }, expectedIndex.start, expectedIndex.numRecords) @@ -225,7 +224,7 @@ func TestTableManager_ForEach(t *testing.T) { } testutil.VerifyIndexes(t, userID, queries, - func(ctx context.Context, table string, callback func(boltdb *bbolt.DB) error) error { + func(ctx context.Context, table string, callback func(b *bbolt.DB) error) error { return tm.ForEach(ctx, table, callback) }, 0, 30) diff --git a/pkg/storage/stores/shipper/index/table_test.go b/pkg/storage/stores/shipper/indexshipper/boltdb/table_test.go similarity index 89% rename from pkg/storage/stores/shipper/index/table_test.go rename to pkg/storage/stores/shipper/indexshipper/boltdb/table_test.go index a5f0fdd5cf5ff..44c583e7966cc 100644 --- a/pkg/storage/stores/shipper/index/table_test.go +++ b/pkg/storage/stores/shipper/indexshipper/boltdb/table_test.go @@ -1,4 +1,4 @@ -package index +package boltdb import ( "context" @@ -15,10 +15,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/chunk/client/util" - shipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/indexfile" - "github.com/grafana/loki/pkg/storage/stores/shipper/testutil" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/testutil" ) const ( @@ -27,21 +26,21 @@ const ( ) type mockIndexShipper struct { - addedIndexes map[string][]shipper_index.Index + addedIndexes map[string][]shipperindex.Index } func newMockIndexShipper() Shipper { return &mockIndexShipper{ - addedIndexes: make(map[string][]shipper_index.Index), + addedIndexes: make(map[string][]shipperindex.Index), } } -func (m *mockIndexShipper) AddIndex(tableName, _ string, index shipper_index.Index) error { +func (m *mockIndexShipper) AddIndex(tableName, _ string, index shipperindex.Index) error { m.addedIndexes[tableName] = append(m.addedIndexes[tableName], index) return nil } -func (m *mockIndexShipper) ForEach(_ context.Context, tableName, _ string, callback shipper_index.ForEachIndexCallback) error { +func (m *mockIndexShipper) ForEach(_ context.Context, tableName, _ string, callback shipperindex.ForEachIndexCallback) error { for _, idx := range m.addedIndexes[tableName] { if err := callback(false, idx); err != nil { return err @@ -111,7 +110,7 @@ func TestLoadTable(t *testing.T) { require.Error(t, err) // try loading the table. - table, err := LoadTable(tablePath, "test", newMockIndexShipper(), false, newMetrics(nil)) + table, err := LoadTable(tablePath, "test", newMockIndexShipper(), false, newTableManagerMetrics(nil)) require.NoError(t, err) require.NotNil(t, table) @@ -126,7 +125,7 @@ func TestLoadTable(t *testing.T) { // query the loaded table to see if it has right data. require.NoError(t, table.Snapshot()) - testutil.VerifyIndexes(t, userID, []index.Query{{TableName: table.name}}, func(ctx context.Context, _ string, callback func(boltdb *bbolt.DB) error) error { + testutil.VerifyIndexes(t, userID, []index.Query{{TableName: table.name}}, func(ctx context.Context, _ string, callback func(b *bbolt.DB) error) error { return table.ForEach(ctx, callback) }, 0, 20) } @@ -187,7 +186,7 @@ func TestTable_Write(t *testing.T) { // test that the table has current + previous records testutil.VerifyIndexes(t, userID, []index.Query{{}}, - func(ctx context.Context, _ string, callback func(boltdb *bbolt.DB) error) error { + func(ctx context.Context, _ string, callback func(b *bbolt.DB) error) error { return table.ForEach(ctx, callback) }, 0, (i+1)*10) @@ -227,9 +226,9 @@ func TestTable_HandoverIndexesToShipper(t *testing.T) { require.Len(t, indexShipper.addedIndexes[table.name], 1) testutil.VerifyIndexes(t, userID, []index.Query{{TableName: table.name}}, - func(ctx context.Context, _ string, callback func(boltdb *bbolt.DB) error) error { - return indexShipper.ForEach(ctx, table.name, "", func(_ bool, index shipper_index.Index) error { - return callback(index.(*indexfile.IndexFile).GetBoltDB()) + func(ctx context.Context, _ string, callback func(b *bbolt.DB) error) error { + return indexShipper.ForEach(ctx, table.name, "", func(_ bool, index shipperindex.Index) error { + return callback(index.(*IndexFile).GetBoltDB()) }) }, 0, 10) @@ -247,9 +246,9 @@ func TestTable_HandoverIndexesToShipper(t *testing.T) { // check that shipper got the new data we handed over require.Len(t, indexShipper.addedIndexes[table.name], 2) testutil.VerifyIndexes(t, userID, []index.Query{{TableName: table.name}}, - func(ctx context.Context, _ string, callback func(boltdb *bbolt.DB) error) error { - return indexShipper.ForEach(ctx, table.name, "", func(_ bool, index shipper_index.Index) error { - return callback(index.(*indexfile.IndexFile).GetBoltDB()) + func(ctx context.Context, _ string, callback func(b *bbolt.DB) error) error { + return indexShipper.ForEach(ctx, table.name, "", func(_ bool, index shipperindex.Index) error { + return callback(index.(*IndexFile).GetBoltDB()) }) }, 0, 20) @@ -268,7 +267,7 @@ func Test_LoadBoltDBsFromDir(t *testing.T) { NumRecords: 10, }, }, - "db1" + indexfile.TempFileSuffix: { // a snapshot file which should be ignored. + "db1" + TempFileSuffix: { // a snapshot file which should be ignored. DBRecords: testutil.DBRecords{ Start: 0, NumRecords: 10, @@ -288,7 +287,7 @@ func Test_LoadBoltDBsFromDir(t *testing.T) { require.NoError(t, db.Close()) // try loading the dbs - dbs, err := loadBoltDBsFromDir(tablePath, newMetrics(nil)) + dbs, err := loadBoltDBsFromDir(tablePath, newTableManagerMetrics(nil)) require.NoError(t, err) // check that we have just 2 dbs @@ -297,8 +296,8 @@ func Test_LoadBoltDBsFromDir(t *testing.T) { require.NotNil(t, dbs["db2"]) // close all the open dbs - for _, boltdb := range dbs { - require.NoError(t, boltdb.Close()) + for _, db := range dbs { + require.NoError(t, db.Close()) } dirEntries, err := os.ReadDir(tablePath) @@ -336,7 +335,7 @@ func TestTable_ImmutableUploads(t *testing.T) { tableName := "test-table" tablePath := testutil.SetupDBsAtPath(t, filepath.Join(indexPath, tableName), dbs, nil) - table, err := LoadTable(tablePath, "test", indexShipper, false, newMetrics(nil)) + table, err := LoadTable(tablePath, "test", indexShipper, false, newTableManagerMetrics(nil)) require.NoError(t, err) require.NotNil(t, table) @@ -371,7 +370,7 @@ func TestTable_ImmutableUploads(t *testing.T) { } // clear dbs handed over to shipper - mockIndexShipper.addedIndexes = map[string][]shipper_index.Index{} + mockIndexShipper.addedIndexes = map[string][]shipperindex.Index{} // force handover of dbs require.NoError(t, table.HandoverIndexesToShipper(true)) @@ -423,7 +422,7 @@ func TestTable_MultiQueries(t *testing.T) { }, []byte(user1)) // try loading the table. - table, err := LoadTable(tablePath, "test", newMockIndexShipper(), false, newMetrics(nil)) + table, err := LoadTable(tablePath, "test", newMockIndexShipper(), false, newTableManagerMetrics(nil)) require.NoError(t, err) require.NotNil(t, table) defer func() { @@ -440,14 +439,14 @@ func TestTable_MultiQueries(t *testing.T) { // querying data for user1 should return both data from common index and user1's index testutil.VerifyIndexes(t, user1, queries, - func(ctx context.Context, _ string, callback func(boltdb *bbolt.DB) error) error { + func(ctx context.Context, _ string, callback func(b *bbolt.DB) error) error { return table.ForEach(ctx, callback) }, 5, 30) // querying data for user2 should return only common index testutil.VerifyIndexes(t, user2, queries, - func(ctx context.Context, _ string, callback func(boltdb *bbolt.DB) error) error { + func(ctx context.Context, _ string, callback func(b *bbolt.DB) error) error { return table.ForEach(ctx, callback) }, 5, 15) diff --git a/pkg/storage/stores/indexshipper/compactor/client/grpc.go b/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc.go similarity index 93% rename from pkg/storage/stores/indexshipper/compactor/client/grpc.go rename to pkg/storage/stores/shipper/indexshipper/compactor/client/grpc.go index 1e1bf3135230a..4716b177acfee 100644 --- a/pkg/storage/stores/indexshipper/compactor/client/grpc.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc.go @@ -12,8 +12,8 @@ import ( "github.com/prometheus/common/model" "google.golang.org/grpc" - deletion_grpc "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/client/grpc" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" + deletion_grpc "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" ) type GRPCConfig struct { diff --git a/pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.pb.go b/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.pb.go similarity index 92% rename from pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.pb.go rename to pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.pb.go index 533a2ecfd6316..df8587446e890 100644 --- a/pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.pb.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.proto +// source: pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.proto package grpc @@ -34,7 +34,7 @@ type GetDeleteRequestsRequest struct { func (m *GetDeleteRequestsRequest) Reset() { *m = GetDeleteRequestsRequest{} } func (*GetDeleteRequestsRequest) ProtoMessage() {} func (*GetDeleteRequestsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_af4e385854eeb93d, []int{0} + return fileDescriptor_5620dfc5f1418630, []int{0} } func (m *GetDeleteRequestsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -70,7 +70,7 @@ type GetDeleteRequestsResponse struct { func (m *GetDeleteRequestsResponse) Reset() { *m = GetDeleteRequestsResponse{} } func (*GetDeleteRequestsResponse) ProtoMessage() {} func (*GetDeleteRequestsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_af4e385854eeb93d, []int{1} + return fileDescriptor_5620dfc5f1418630, []int{1} } func (m *GetDeleteRequestsResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -118,7 +118,7 @@ type DeleteRequest struct { func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (*DeleteRequest) ProtoMessage() {} func (*DeleteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_af4e385854eeb93d, []int{2} + return fileDescriptor_5620dfc5f1418630, []int{2} } func (m *DeleteRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -195,7 +195,7 @@ type GetCacheGenNumbersRequest struct { func (m *GetCacheGenNumbersRequest) Reset() { *m = GetCacheGenNumbersRequest{} } func (*GetCacheGenNumbersRequest) ProtoMessage() {} func (*GetCacheGenNumbersRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_af4e385854eeb93d, []int{3} + return fileDescriptor_5620dfc5f1418630, []int{3} } func (m *GetCacheGenNumbersRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -231,7 +231,7 @@ type GetCacheGenNumbersResponse struct { func (m *GetCacheGenNumbersResponse) Reset() { *m = GetCacheGenNumbersResponse{} } func (*GetCacheGenNumbersResponse) ProtoMessage() {} func (*GetCacheGenNumbersResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_af4e385854eeb93d, []int{4} + return fileDescriptor_5620dfc5f1418630, []int{4} } func (m *GetCacheGenNumbersResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -276,36 +276,36 @@ func init() { } func init() { - proto.RegisterFile("pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.proto", fileDescriptor_af4e385854eeb93d) -} - -var fileDescriptor_af4e385854eeb93d = []byte{ - // 391 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0xbb, 0x4e, 0xe3, 0x40, - 0x14, 0xf5, 0x6c, 0x1e, 0xab, 0xcc, 0x6a, 0x77, 0xc5, 0x80, 0xd0, 0x60, 0xd0, 0x60, 0xb9, 0x72, - 0x15, 0x4b, 0x81, 0x8e, 0x06, 0x48, 0x44, 0x44, 0x43, 0x61, 0x45, 0x82, 0xd6, 0xb1, 0xaf, 0x12, - 0x8b, 0xc4, 0x76, 0x66, 0xc6, 0x12, 0x74, 0x7c, 0x02, 0x9f, 0x41, 0x4b, 0xcb, 0x17, 0x50, 0xa6, - 0x4c, 0x49, 0x9c, 0x86, 0x32, 0x9f, 0x80, 0x32, 0x76, 0x12, 0xe5, 0xd5, 0x78, 0xe6, 0x9e, 0x73, - 0xe6, 0x3e, 0xce, 0x35, 0xbe, 0x8c, 0x1f, 0x3b, 0xb6, 0x90, 0x11, 0x77, 0x3b, 0xa0, 0x4e, 0x10, - 0x76, 0x10, 0xfa, 0xf0, 0x24, 0xba, 0x41, 0x1c, 0x03, 0xb7, 0xbd, 0xa8, 0x1f, 0xbb, 0x9e, 0x8c, - 0xb8, 0xed, 0xf5, 0x02, 0x08, 0xa5, 0xdd, 0xe1, 0xb1, 0xa7, 0x3e, 0xd5, 0x98, 0x47, 0x32, 0x22, - 0xc5, 0xd9, 0xdd, 0xd4, 0x31, 0x6d, 0x82, 0x6c, 0x40, 0x0f, 0x24, 0x38, 0x30, 0x48, 0x40, 0x48, - 0x91, 0x9f, 0xe6, 0x03, 0x3e, 0xda, 0xc2, 0x89, 0x38, 0x0a, 0x05, 0x90, 0x0b, 0xfc, 0xcf, 0x5f, - 0x61, 0x28, 0x32, 0x0a, 0xd6, 0x9f, 0xda, 0x7e, 0x55, 0xd5, 0x58, 0x79, 0xe5, 0xac, 0x49, 0xcd, - 0x77, 0x84, 0xff, 0xae, 0x28, 0xc8, 0x09, 0xae, 0xf0, 0xec, 0x7a, 0xdb, 0xa0, 0xc8, 0x40, 0x56, - 0xc5, 0x59, 0x02, 0x33, 0x56, 0x48, 0x97, 0xcb, 0x56, 0xd0, 0x07, 0xfa, 0xcb, 0x40, 0x56, 0xc1, - 0x59, 0x02, 0x84, 0xe2, 0xdf, 0x10, 0xfa, 0x8a, 0x2b, 0x28, 0x6e, 0x1e, 0x92, 0x03, 0x5c, 0x1a, - 0x24, 0xc0, 0x9f, 0x69, 0x51, 0x65, 0xcc, 0x02, 0x72, 0x88, 0xcb, 0x42, 0xba, 0x32, 0x11, 0xb4, - 0xa4, 0xe0, 0x3c, 0x9a, 0x55, 0xf1, 0x38, 0xb8, 0x12, 0xfc, 0x2b, 0x49, 0xcb, 0x59, 0x95, 0x05, - 0x60, 0x1e, 0x2b, 0x37, 0xea, 0xae, 0xd7, 0x85, 0x26, 0x84, 0x77, 0x49, 0xbf, 0x0d, 0x7c, 0x61, - 0xd5, 0x0d, 0xd6, 0xb7, 0x91, 0xb9, 0x57, 0x16, 0xfe, 0xcf, 0x41, 0x24, 0x3d, 0x29, 0xe6, 0x8a, - 0x7c, 0xc4, 0x75, 0xb8, 0xf6, 0x81, 0x70, 0xa5, 0x3e, 0xdf, 0x1c, 0x69, 0xe1, 0xbd, 0x8d, 0x05, - 0x10, 0x96, 0x19, 0xbc, 0x6b, 0x6b, 0xfa, 0xe9, 0x4e, 0x3e, 0xef, 0xe6, 0x1e, 0x93, 0xcd, 0x5e, - 0xc9, 0xf2, 0xd9, 0xf6, 0x11, 0x75, 0x63, 0xb7, 0x20, 0x4b, 0x7c, 0x7d, 0x3e, 0x1c, 0x33, 0x6d, - 0x34, 0x66, 0xda, 0x74, 0xcc, 0xd0, 0x4b, 0xca, 0xd0, 0x5b, 0xca, 0xd0, 0x67, 0xca, 0xd0, 0x30, - 0x65, 0xe8, 0x2b, 0x65, 0xe8, 0x3b, 0x65, 0xda, 0x34, 0x65, 0xe8, 0x75, 0xc2, 0xb4, 0xe1, 0x84, - 0x69, 0xa3, 0x09, 0xd3, 0xda, 0x65, 0xf5, 0x3b, 0x9e, 0xfd, 0x04, 0x00, 0x00, 0xff, 0xff, 0x37, - 0x5f, 0x43, 0x54, 0xd2, 0x02, 0x00, 0x00, + proto.RegisterFile("pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.proto", fileDescriptor_5620dfc5f1418630) +} + +var fileDescriptor_5620dfc5f1418630 = []byte{ + // 393 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0x3d, 0x4f, 0xe3, 0x30, + 0x18, 0x8e, 0xaf, 0x1f, 0xa7, 0xfa, 0x74, 0x77, 0x3a, 0xdf, 0xe9, 0x64, 0x02, 0x32, 0x51, 0xa6, + 0x4c, 0x8d, 0x54, 0xd8, 0x98, 0xa0, 0x15, 0x85, 0x85, 0x21, 0xaa, 0x04, 0x6b, 0x9a, 0xbc, 0x6a, + 0x23, 0xda, 0x24, 0xb5, 0x1d, 0x09, 0x36, 0x7e, 0x02, 0x3f, 0x83, 0x95, 0x95, 0x5f, 0xc0, 0xd8, + 0xb1, 0x23, 0x4d, 0x17, 0xc6, 0xfe, 0x04, 0x54, 0x27, 0x69, 0xd5, 0xaf, 0x25, 0xf6, 0xf3, 0x61, + 0xbf, 0x7e, 0x9f, 0x37, 0xf8, 0x2a, 0xbe, 0xef, 0xd9, 0x42, 0x46, 0xdc, 0xed, 0x81, 0x5a, 0x41, + 0xd8, 0xa2, 0x1f, 0xc4, 0x31, 0x70, 0x3b, 0x08, 0x7d, 0x78, 0x28, 0x80, 0x17, 0x0d, 0x63, 0xd7, + 0x93, 0x11, 0xb7, 0xbd, 0x41, 0x00, 0xa1, 0xb4, 0x7b, 0x3c, 0xf6, 0xd4, 0xa7, 0x1e, 0xf3, 0x48, + 0x46, 0xa4, 0xbc, 0xd8, 0x9b, 0x3a, 0xa6, 0x6d, 0x90, 0x2d, 0x18, 0x80, 0x04, 0x07, 0x46, 0x09, + 0x08, 0x29, 0xf2, 0xd5, 0xbc, 0xc3, 0x07, 0x3b, 0x34, 0x11, 0x47, 0xa1, 0x00, 0x72, 0x86, 0x7f, + 0xf9, 0x6b, 0x0a, 0x45, 0x46, 0xc9, 0xfa, 0xd1, 0xf8, 0x5b, 0x57, 0x35, 0xd6, 0x4e, 0x39, 0x1b, + 0x56, 0xf3, 0x15, 0xe1, 0x9f, 0x6b, 0x0e, 0x72, 0x84, 0x6b, 0x3c, 0xdb, 0x5e, 0xb7, 0x28, 0x32, + 0x90, 0x55, 0x73, 0x56, 0xc4, 0x42, 0x15, 0xd2, 0xe5, 0xb2, 0x13, 0x0c, 0x81, 0x7e, 0x33, 0x90, + 0x55, 0x72, 0x56, 0x04, 0xa1, 0xf8, 0x3b, 0x84, 0xbe, 0xd2, 0x4a, 0x4a, 0x2b, 0x20, 0xf9, 0x87, + 0x2b, 0xa3, 0x04, 0xf8, 0x23, 0x2d, 0xab, 0x1b, 0x33, 0x40, 0xfe, 0xe3, 0xaa, 0x90, 0xae, 0x4c, + 0x04, 0xad, 0x28, 0x3a, 0x47, 0x8b, 0x2a, 0x1e, 0x07, 0x57, 0x82, 0x7f, 0x2e, 0x69, 0x35, 0xab, + 0xb2, 0x24, 0xcc, 0x43, 0x95, 0x46, 0xd3, 0xf5, 0xfa, 0xd0, 0x86, 0xf0, 0x26, 0x19, 0x76, 0x81, + 0x2f, 0xa3, 0xba, 0xc4, 0xfa, 0x2e, 0x31, 0xcf, 0xca, 0xc2, 0xbf, 0x39, 0x88, 0x64, 0x20, 0x45, + 0xe1, 0xc8, 0x5b, 0xdc, 0xa4, 0x1b, 0x6f, 0x08, 0xd7, 0x9a, 0xc5, 0xe4, 0x48, 0x07, 0xff, 0xd9, + 0x1a, 0x00, 0x61, 0x59, 0xc0, 0xfb, 0xa6, 0xa6, 0x1f, 0xef, 0xd5, 0xf3, 0xd7, 0xdc, 0x62, 0xb2, + 0xfd, 0x56, 0xb2, 0x3a, 0xb6, 0xbb, 0x45, 0xdd, 0xd8, 0x6f, 0xc8, 0x2e, 0xbe, 0x38, 0x1d, 0x4f, + 0x99, 0x36, 0x99, 0x32, 0x6d, 0x3e, 0x65, 0xe8, 0x29, 0x65, 0xe8, 0x25, 0x65, 0xe8, 0x3d, 0x65, + 0x68, 0x9c, 0x32, 0xf4, 0x91, 0x32, 0xf4, 0x99, 0x32, 0x6d, 0x9e, 0x32, 0xf4, 0x3c, 0x63, 0xda, + 0x78, 0xc6, 0xb4, 0xc9, 0x8c, 0x69, 0xdd, 0xaa, 0xfa, 0x1d, 0x4f, 0xbe, 0x02, 0x00, 0x00, 0xff, + 0xff, 0xd3, 0x51, 0x00, 0xbe, 0xda, 0x02, 0x00, 0x00, } func (this *GetDeleteRequestsRequest) Equal(that interface{}) bool { @@ -619,7 +619,7 @@ var _Compactor_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.proto", + Metadata: "pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.proto", } func (m *GetDeleteRequestsRequest) Marshal() (dAtA []byte, err error) { diff --git a/pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.proto b/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.proto similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/client/grpc/grpc.proto rename to pkg/storage/stores/shipper/indexshipper/compactor/client/grpc/grpc.proto diff --git a/pkg/storage/stores/indexshipper/compactor/client/http.go b/pkg/storage/stores/shipper/indexshipper/compactor/client/http.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/client/http.go rename to pkg/storage/stores/shipper/indexshipper/compactor/client/http.go index 817892df2c61f..3ac2593717340 100644 --- a/pkg/storage/stores/indexshipper/compactor/client/http.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/client/http.go @@ -13,7 +13,7 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/crypto/tls" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/compactor.go b/pkg/storage/stores/shipper/indexshipper/compactor/compactor.go similarity index 97% rename from pkg/storage/stores/indexshipper/compactor/compactor.go rename to pkg/storage/stores/shipper/indexshipper/compactor/compactor.go index 3c8b85ec3a503..e3583d4ea7fda 100644 --- a/pkg/storage/stores/indexshipper/compactor/compactor.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/compactor.go @@ -25,9 +25,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - shipper_storage "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/filter" util_log "github.com/grafana/loki/pkg/util/log" @@ -144,7 +144,7 @@ func (cfg *Config) Validate() error { return errors.New("interval for applying retention should either be set to a 0 or a multiple of compaction interval") } - if err := shipper_storage.ValidateSharedStoreKeyPrefix(cfg.SharedStoreKeyPrefix); err != nil { + if err := storage.ValidateSharedStoreKeyPrefix(cfg.SharedStoreKeyPrefix); err != nil { return err } @@ -159,7 +159,7 @@ type Compactor struct { services.Service cfg Config - indexStorageClient shipper_storage.Client + indexStorageClient storage.Client tableMarker retention.TableMarker sweeper *retention.Sweeper deleteRequestsStore deletion.DeleteRequestsStore @@ -189,7 +189,7 @@ type Compactor struct { type storeContainer struct { tableMarker retention.TableMarker sweeper *retention.Sweeper - indexStorageClient shipper_storage.Client + indexStorageClient storage.Client } type Limits interface { @@ -294,7 +294,7 @@ func (c *Compactor) init(objectStoreClients map[string]client.ObjectClient, sche c.storeContainers = make(map[string]storeContainer, len(objectStoreClients)) for objectStoreType, objectClient := range objectStoreClients { var sc storeContainer - sc.indexStorageClient = shipper_storage.NewIndexStorageClient(objectClient, c.cfg.SharedStoreKeyPrefix) + sc.indexStorageClient = storage.NewIndexStorageClient(objectClient, c.cfg.SharedStoreKeyPrefix) if c.cfg.RetentionEnabled { // given that compaction can now run on multiple object stores, marker files are stored under /retention/{objectStoreType}/markers/ @@ -341,7 +341,7 @@ func (c *Compactor) init(objectStoreClients map[string]client.ObjectClient, sche func (c *Compactor) initDeletes(objectClient client.ObjectClient, r prometheus.Registerer, limits Limits) error { deletionWorkDir := filepath.Join(c.cfg.WorkingDirectory, "deletion") - store, err := deletion.NewDeleteStore(deletionWorkDir, shipper_storage.NewIndexStorageClient(objectClient, c.cfg.SharedStoreKeyPrefix)) + store, err := deletion.NewDeleteStore(deletionWorkDir, storage.NewIndexStorageClient(objectClient, c.cfg.SharedStoreKeyPrefix)) if err != nil { return err } diff --git a/pkg/storage/stores/indexshipper/compactor/compactor_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/compactor_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/compactor_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/compactor_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_request.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_request.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_request.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_request.go index cf7c0d415be9e..c4314ee58865f 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_request.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_request.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/logql/syntax" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" "github.com/grafana/loki/pkg/util/filter" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_request_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_request_test.go similarity index 99% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_request_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_request_test.go index ddb19ee04cc97..7df4763c3309a 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_request_test.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_request_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logql/syntax" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" "github.com/grafana/loki/pkg/util/filter" ) diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_client.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_client.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_client.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_client.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_client_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_client_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_client_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_client_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_manager.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_manager.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_manager.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_manager.go index 079186a5fce4c..5c3e7e95a0237 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_manager.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_manager.go @@ -12,8 +12,8 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" "github.com/grafana/loki/pkg/util/filter" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_manager_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_manager_test.go similarity index 99% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_manager_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_manager_test.go index 228b20f7ca620..012a6d7140416 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_manager_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logql/syntax" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" "github.com/grafana/loki/pkg/util/filter" ) diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_store.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_store.go similarity index 99% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_store.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_store.go index dbcef77f159ea..20212467c15d7 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_store.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_store.go @@ -17,8 +17,8 @@ import ( "github.com/prometheus/common/model" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" "github.com/grafana/loki/pkg/storage/stores/series/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" ) type ( diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_store_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_store_test.go similarity index 99% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_store_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_store_test.go index d56d584d4a21e..0bf4cb27f7bd8 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_store_test.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_store_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" ) func TestDeleteRequestsStore(t *testing.T) { diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_table.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_table.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_table.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_table.go index ae535e31ff452..3143a51cc5444 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_table.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_table.go @@ -15,9 +15,9 @@ import ( "github.com/grafana/loki/pkg/chunkenc" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" "github.com/grafana/loki/pkg/storage/stores/series/index" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_table_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_table_test.go similarity index 93% rename from pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_table_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_table_test.go index 46a4834f59808..4b30a9bc67b1f 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/delete_requests_table_test.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/delete_requests_table_test.go @@ -9,10 +9,10 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/testutil" - "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/testutil" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" ) func TestDeleteRequestsTable(t *testing.T) { diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/grpc_request_handler.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/grpc_request_handler.go similarity index 96% rename from pkg/storage/stores/indexshipper/compactor/deletion/grpc_request_handler.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/grpc_request_handler.go index 15b5452e89d5a..4029895b9d8fd 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/grpc_request_handler.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/grpc_request_handler.go @@ -8,7 +8,7 @@ import ( "github.com/grafana/dskit/tenant" "github.com/pkg/errors" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/client/grpc" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/grpc_request_handler_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/grpc_request_handler_test.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/deletion/grpc_request_handler_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/grpc_request_handler_test.go index 4e0975cc4bd92..8f1f9560de88a 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/grpc_request_handler_test.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/grpc_request_handler_test.go @@ -17,8 +17,8 @@ import ( "google.golang.org/grpc/status" "google.golang.org/grpc/test/bufconn" - compactor_client_grpc "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/client/grpc" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode" + compactor_client_grpc "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/client/grpc" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode" ) func server(t *testing.T, h *GRPCRequestHandler) (compactor_client_grpc.CompactorClient, func()) { diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/metrics.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/metrics.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/metrics.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/metrics.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/noop_delete_requests_store.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/noop_delete_requests_store.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/noop_delete_requests_store.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/noop_delete_requests_store.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/request_handler.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/request_handler.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/request_handler.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/request_handler.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/request_handler_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/request_handler_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/request_handler_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/request_handler_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/tenant_delete_requests_client.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_delete_requests_client.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/tenant_delete_requests_client.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_delete_requests_client.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/tenant_delete_requests_client_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_delete_requests_client_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/tenant_delete_requests_client_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_delete_requests_client_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/tenant_request_handler.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_request_handler.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/tenant_request_handler.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_request_handler.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/tenant_request_handler_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_request_handler_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/tenant_request_handler_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/tenant_request_handler_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/util.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/util.go similarity index 91% rename from pkg/storage/stores/indexshipper/compactor/deletion/util.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/util.go index 405041fac8e79..f5165ec07f508 100644 --- a/pkg/storage/stores/indexshipper/compactor/deletion/util.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/util.go @@ -3,9 +3,8 @@ package deletion import ( "errors" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode" - "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode" ) var ( diff --git a/pkg/storage/stores/indexshipper/compactor/deletion/util_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletion/util_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletion/util_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletion/util_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletionmode/mode.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode/mode.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletionmode/mode.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletionmode/mode.go diff --git a/pkg/storage/stores/indexshipper/compactor/deletionmode/mode_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode/mode_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/deletionmode/mode_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/deletionmode/mode_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/generationnumber/gennumber_loader.go b/pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/gennumber_loader.go similarity index 99% rename from pkg/storage/stores/indexshipper/compactor/generationnumber/gennumber_loader.go rename to pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/gennumber_loader.go index c88f3bd0ce3cf..ec8bb0bcab567 100644 --- a/pkg/storage/stores/indexshipper/compactor/generationnumber/gennumber_loader.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/gennumber_loader.go @@ -8,10 +8,9 @@ import ( "time" "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" "github.com/grafana/loki/pkg/util/log" - - "github.com/prometheus/client_golang/prometheus" ) const reloadDuration = 5 * time.Minute diff --git a/pkg/storage/stores/indexshipper/compactor/generationnumber/gennumber_loader_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/gennumber_loader_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/generationnumber/gennumber_loader_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/gennumber_loader_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/generationnumber/metrics.go b/pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/metrics.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/generationnumber/metrics.go rename to pkg/storage/stores/shipper/indexshipper/compactor/generationnumber/metrics.go diff --git a/pkg/storage/stores/indexshipper/compactor/index_set.go b/pkg/storage/stores/shipper/indexshipper/compactor/index_set.go similarity index 97% rename from pkg/storage/stores/indexshipper/compactor/index_set.go rename to pkg/storage/stores/shipper/indexshipper/compactor/index_set.go index 2224afa296a63..28004011966a7 100644 --- a/pkg/storage/stores/indexshipper/compactor/index_set.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/index_set.go @@ -14,9 +14,9 @@ import ( "github.com/grafana/loki/pkg/chunkenc" "github.com/grafana/loki/pkg/storage/chunk/client/util" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/metrics.go b/pkg/storage/stores/shipper/indexshipper/compactor/metrics.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/metrics.go rename to pkg/storage/stores/shipper/indexshipper/compactor/metrics.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/expiration.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/expiration.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/expiration.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/expiration.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/expiration_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/expiration_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/expiration_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/expiration_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/marker.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/marker.go similarity index 99% rename from pkg/storage/stores/indexshipper/compactor/retention/marker.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/marker.go index ca3d4c5bf5a91..84e461cd7bc5d 100644 --- a/pkg/storage/stores/indexshipper/compactor/retention/marker.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/retention/marker.go @@ -16,7 +16,7 @@ import ( "go.etcd.io/bbolt" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/retention/marker_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/marker_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/marker_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/marker_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/metrics.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/metrics.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/metrics.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/metrics.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/pool.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/pool.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/pool.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/pool.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/retention.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/retention.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/retention.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/retention.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/retention_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/retention_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/retention_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/retention_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/series.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/series.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/series.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/series.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/series_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/series_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/series_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/series_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/util.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/util.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/util.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/util.go diff --git a/pkg/storage/stores/indexshipper/compactor/retention/util_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/retention/util_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/compactor/retention/util_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/retention/util_test.go diff --git a/pkg/storage/stores/indexshipper/compactor/table.go b/pkg/storage/stores/shipper/indexshipper/compactor/table.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/table.go rename to pkg/storage/stores/shipper/indexshipper/compactor/table.go index 95d6b3f31b526..01dcd340bb61f 100644 --- a/pkg/storage/stores/indexshipper/compactor/table.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/table.go @@ -14,8 +14,8 @@ import ( chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/compactor/table_test.go b/pkg/storage/stores/shipper/indexshipper/compactor/table_test.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/table_test.go rename to pkg/storage/stores/shipper/indexshipper/compactor/table_test.go index 3c93ad0929954..552e911690ca8 100644 --- a/pkg/storage/stores/indexshipper/compactor/table_test.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/table_test.go @@ -14,8 +14,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" ) const ( diff --git a/pkg/storage/stores/indexshipper/compactor/testutil.go b/pkg/storage/stores/shipper/indexshipper/compactor/testutil.go similarity index 98% rename from pkg/storage/stores/indexshipper/compactor/testutil.go rename to pkg/storage/stores/shipper/indexshipper/compactor/testutil.go index cf2b6d15e56bb..b79be210ec944 100644 --- a/pkg/storage/stores/indexshipper/compactor/testutil.go +++ b/pkg/storage/stores/shipper/indexshipper/compactor/testutil.go @@ -20,9 +20,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/testutil" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/testutil" ) const ( diff --git a/pkg/storage/stores/indexshipper/downloads/index_set.go b/pkg/storage/stores/shipper/indexshipper/downloads/index_set.go similarity index 98% rename from pkg/storage/stores/indexshipper/downloads/index_set.go rename to pkg/storage/stores/shipper/indexshipper/downloads/index_set.go index 6da3774c1c710..1a2cefb491cc0 100644 --- a/pkg/storage/stores/indexshipper/downloads/index_set.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/index_set.go @@ -17,8 +17,8 @@ import ( "golang.org/x/sync/errgroup" "github.com/grafana/loki/pkg/storage/chunk/client/util" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/util/spanlogger" ) diff --git a/pkg/storage/stores/indexshipper/downloads/index_set_test.go b/pkg/storage/stores/shipper/indexshipper/downloads/index_set_test.go similarity index 97% rename from pkg/storage/stores/indexshipper/downloads/index_set_test.go rename to pkg/storage/stores/shipper/indexshipper/downloads/index_set_test.go index 660d0170e87f9..a0cadf1da6cdb 100644 --- a/pkg/storage/stores/indexshipper/downloads/index_set_test.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/index_set_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/storage/chunk/client/util" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/downloads/metrics.go b/pkg/storage/stores/shipper/indexshipper/downloads/metrics.go similarity index 100% rename from pkg/storage/stores/indexshipper/downloads/metrics.go rename to pkg/storage/stores/shipper/indexshipper/downloads/metrics.go diff --git a/pkg/storage/stores/indexshipper/downloads/table.go b/pkg/storage/stores/shipper/indexshipper/downloads/table.go similarity index 98% rename from pkg/storage/stores/indexshipper/downloads/table.go rename to pkg/storage/stores/shipper/indexshipper/downloads/table.go index c067ef3e7e216..6e4d8093b7e43 100644 --- a/pkg/storage/stores/indexshipper/downloads/table.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/table.go @@ -15,8 +15,8 @@ import ( "golang.org/x/sync/errgroup" "github.com/grafana/loki/pkg/storage/chunk/client/util" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/util/spanlogger" ) diff --git a/pkg/storage/stores/indexshipper/downloads/table_manager.go b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go similarity index 98% rename from pkg/storage/stores/indexshipper/downloads/table_manager.go rename to pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go index 9904d23706045..c78baab0c57f8 100644 --- a/pkg/storage/stores/indexshipper/downloads/table_manager.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager.go @@ -16,9 +16,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletion" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletion" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" "github.com/grafana/loki/pkg/validation" ) diff --git a/pkg/storage/stores/indexshipper/downloads/table_manager_test.go b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go similarity index 98% rename from pkg/storage/stores/indexshipper/downloads/table_manager_test.go rename to pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go index 4e05c91854572..b5b73bb421887 100644 --- a/pkg/storage/stores/indexshipper/downloads/table_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/table_manager_test.go @@ -13,8 +13,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" "github.com/grafana/loki/pkg/validation" ) diff --git a/pkg/storage/stores/indexshipper/downloads/table_test.go b/pkg/storage/stores/shipper/indexshipper/downloads/table_test.go similarity index 99% rename from pkg/storage/stores/indexshipper/downloads/table_test.go rename to pkg/storage/stores/shipper/indexshipper/downloads/table_test.go index 02ad12e310449..cc88451704b65 100644 --- a/pkg/storage/stores/indexshipper/downloads/table_test.go +++ b/pkg/storage/stores/shipper/indexshipper/downloads/table_test.go @@ -12,8 +12,8 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/downloads/testutil.go b/pkg/storage/stores/shipper/indexshipper/downloads/testutil.go similarity index 100% rename from pkg/storage/stores/indexshipper/downloads/testutil.go rename to pkg/storage/stores/shipper/indexshipper/downloads/testutil.go diff --git a/pkg/storage/stores/indexshipper/downloads/util.go b/pkg/storage/stores/shipper/indexshipper/downloads/util.go similarity index 100% rename from pkg/storage/stores/indexshipper/downloads/util.go rename to pkg/storage/stores/shipper/indexshipper/downloads/util.go diff --git a/pkg/storage/stores/indexshipper/gatewayclient/gateway_client.go b/pkg/storage/stores/shipper/indexshipper/gatewayclient/gateway_client.go similarity index 98% rename from pkg/storage/stores/indexshipper/gatewayclient/gateway_client.go rename to pkg/storage/stores/shipper/indexshipper/gatewayclient/gateway_client.go index fd1fbe66fdf6e..a82ef8d2d6388 100644 --- a/pkg/storage/stores/indexshipper/gatewayclient/gateway_client.go +++ b/pkg/storage/stores/shipper/indexshipper/gatewayclient/gateway_client.go @@ -22,8 +22,7 @@ import ( "github.com/grafana/loki/pkg/distributor/clientpool" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" util_log "github.com/grafana/loki/pkg/util/log" util_math "github.com/grafana/loki/pkg/util/math" ) @@ -279,7 +278,7 @@ func (s *GatewayClient) doQueries(ctx context.Context, queries []index.Query, ca gatewayQueries := make([]*logproto.IndexQuery, 0, len(queries)) for _, query := range queries { - queryKeyQueryMap[shipper_util.QueryKey(query)] = query + queryKeyQueryMap[index.QueryKey(query)] = query gatewayQueries = append(gatewayQueries, &logproto.IndexQuery{ TableName: query.TableName, HashValue: query.HashValue, diff --git a/pkg/storage/stores/indexshipper/gatewayclient/gateway_client_test.go b/pkg/storage/stores/shipper/indexshipper/gatewayclient/gateway_client_test.go similarity index 98% rename from pkg/storage/stores/indexshipper/gatewayclient/gateway_client_test.go rename to pkg/storage/stores/shipper/indexshipper/gatewayclient/gateway_client_test.go index b16916ebd1dd2..59b5826863360 100644 --- a/pkg/storage/stores/indexshipper/gatewayclient/gateway_client_test.go +++ b/pkg/storage/stores/shipper/indexshipper/gatewayclient/gateway_client_test.go @@ -22,8 +22,7 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" - "github.com/grafana/loki/pkg/storage/stores/shipper/util" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" "github.com/grafana/loki/pkg/validation" ) @@ -78,7 +77,7 @@ func (m mockIndexGatewayServer) QueryIndex(request *logproto.QueryIndexRequest, }) } - resp.QueryKey = util.QueryKey(index.Query{ + resp.QueryKey = index.QueryKey(index.Query{ TableName: query.TableName, HashValue: query.HashValue, RangeValuePrefix: query.RangeValuePrefix, diff --git a/pkg/storage/stores/indexshipper/gatewayclient/index_gateway_grpc_pool.go b/pkg/storage/stores/shipper/indexshipper/gatewayclient/index_gateway_grpc_pool.go similarity index 100% rename from pkg/storage/stores/indexshipper/gatewayclient/index_gateway_grpc_pool.go rename to pkg/storage/stores/shipper/indexshipper/gatewayclient/index_gateway_grpc_pool.go diff --git a/pkg/storage/stores/indexshipper/index/index.go b/pkg/storage/stores/shipper/indexshipper/index/index.go similarity index 100% rename from pkg/storage/stores/indexshipper/index/index.go rename to pkg/storage/stores/shipper/indexshipper/index/index.go diff --git a/pkg/storage/stores/shipper/indexgateway/config.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/config.go similarity index 100% rename from pkg/storage/stores/shipper/indexgateway/config.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/config.go diff --git a/pkg/storage/stores/shipper/indexgateway/gateway.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/gateway.go similarity index 98% rename from pkg/storage/stores/shipper/indexgateway/gateway.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/gateway.go index 565712c59ca0f..1120e94c6eabc 100644 --- a/pkg/storage/stores/shipper/indexgateway/gateway.go +++ b/pkg/storage/stores/shipper/indexshipper/indexgateway/gateway.go @@ -22,7 +22,6 @@ import ( "github.com/grafana/loki/pkg/storage/stores" "github.com/grafana/loki/pkg/storage/stores/index" seriesindex "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/util" "github.com/grafana/loki/pkg/util/spanlogger" ) @@ -159,7 +158,7 @@ func buildResponses(query seriesindex.Query, batch seriesindex.ReadBatchResult, for itr.Next() { if len(resp) == maxIndexEntriesPerResponse { err := callback(&logproto.QueryIndexResponse{ - QueryKey: util.QueryKey(query), + QueryKey: seriesindex.QueryKey(query), Rows: resp, }) if err != nil { @@ -176,7 +175,7 @@ func buildResponses(query seriesindex.Query, batch seriesindex.ReadBatchResult, if len(resp) != 0 { err := callback(&logproto.QueryIndexResponse{ - QueryKey: util.QueryKey(query), + QueryKey: seriesindex.QueryKey(query), Rows: resp, }) if err != nil { diff --git a/pkg/storage/stores/shipper/indexgateway/gateway_test.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/gateway_test.go similarity index 98% rename from pkg/storage/stores/shipper/indexgateway/gateway_test.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/gateway_test.go index 0e3a0f6df2d27..192061038a33e 100644 --- a/pkg/storage/stores/shipper/indexgateway/gateway_test.go +++ b/pkg/storage/stores/shipper/indexshipper/indexgateway/gateway_test.go @@ -17,7 +17,6 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/util" util_test "github.com/grafana/loki/pkg/util" util_log "github.com/grafana/loki/pkg/util/log" util_math "github.com/grafana/loki/pkg/util/math" @@ -137,7 +136,7 @@ func TestGateway_QueryIndex(t *testing.T) { end: util_math.Min(j+maxIndexEntriesPerResponse, responseSize), }) } - expectedQueryKey = util.QueryKey(query) + expectedQueryKey = index.QueryKey(query) gateway.indexClients = []IndexClientWithRange{{ IndexClient: &mockIndexClient{response: &mockBatch{size: responseSize}}, TableRange: config.TableRange{ @@ -173,7 +172,7 @@ func TestGateway_QueryIndex_multistore(t *testing.T) { var server logproto.IndexGateway_QueryIndexServer = &mockQueryIndexServer{ callback: func(resp *logproto.QueryIndexResponse) { require.True(t, len(expectedQueries) > 0) - require.Equal(t, util.QueryKey(index.Query{ + require.Equal(t, index.QueryKey(index.Query{ TableName: expectedQueries[0].TableName, HashValue: expectedQueries[0].HashValue, RangeValuePrefix: expectedQueries[0].RangeValuePrefix, diff --git a/pkg/storage/stores/shipper/indexgateway/grpc.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/grpc.go similarity index 100% rename from pkg/storage/stores/shipper/indexgateway/grpc.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/grpc.go diff --git a/pkg/storage/stores/shipper/indexgateway/lifecycle.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/lifecycle.go similarity index 100% rename from pkg/storage/stores/shipper/indexgateway/lifecycle.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/lifecycle.go diff --git a/pkg/storage/stores/shipper/indexgateway/ringmanager.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/ringmanager.go similarity index 100% rename from pkg/storage/stores/shipper/indexgateway/ringmanager.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/ringmanager.go diff --git a/pkg/storage/stores/shipper/indexgateway/shufflesharding.go b/pkg/storage/stores/shipper/indexshipper/indexgateway/shufflesharding.go similarity index 100% rename from pkg/storage/stores/shipper/indexgateway/shufflesharding.go rename to pkg/storage/stores/shipper/indexshipper/indexgateway/shufflesharding.go diff --git a/pkg/storage/stores/indexshipper/shipper.go b/pkg/storage/stores/shipper/indexshipper/shipper.go similarity index 96% rename from pkg/storage/stores/indexshipper/shipper.go rename to pkg/storage/stores/shipper/indexshipper/shipper.go index 7bba6096482c9..b4c29a7520339 100644 --- a/pkg/storage/stores/indexshipper/shipper.go +++ b/pkg/storage/stores/shipper/indexshipper/shipper.go @@ -17,11 +17,11 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/downloads" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/gatewayclient" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/uploads" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/downloads" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/gatewayclient" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/uploads" ) type Mode string diff --git a/pkg/storage/stores/indexshipper/storage/cached_client.go b/pkg/storage/stores/shipper/indexshipper/storage/cached_client.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/cached_client.go rename to pkg/storage/stores/shipper/indexshipper/storage/cached_client.go diff --git a/pkg/storage/stores/indexshipper/storage/cached_client_test.go b/pkg/storage/stores/shipper/indexshipper/storage/cached_client_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/cached_client_test.go rename to pkg/storage/stores/shipper/indexshipper/storage/cached_client_test.go diff --git a/pkg/storage/stores/indexshipper/storage/client.go b/pkg/storage/stores/shipper/indexshipper/storage/client.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/client.go rename to pkg/storage/stores/shipper/indexshipper/storage/client.go diff --git a/pkg/storage/stores/indexshipper/storage/client_test.go b/pkg/storage/stores/shipper/indexshipper/storage/client_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/client_test.go rename to pkg/storage/stores/shipper/indexshipper/storage/client_test.go diff --git a/pkg/storage/stores/indexshipper/storage/index_set.go b/pkg/storage/stores/shipper/indexshipper/storage/index_set.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/index_set.go rename to pkg/storage/stores/shipper/indexshipper/storage/index_set.go diff --git a/pkg/storage/stores/indexshipper/storage/prefixed_object_client.go b/pkg/storage/stores/shipper/indexshipper/storage/prefixed_object_client.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/prefixed_object_client.go rename to pkg/storage/stores/shipper/indexshipper/storage/prefixed_object_client.go diff --git a/pkg/storage/stores/indexshipper/storage/util.go b/pkg/storage/stores/shipper/indexshipper/storage/util.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/util.go rename to pkg/storage/stores/shipper/indexshipper/storage/util.go diff --git a/pkg/storage/stores/indexshipper/storage/util_test.go b/pkg/storage/stores/shipper/indexshipper/storage/util_test.go similarity index 100% rename from pkg/storage/stores/indexshipper/storage/util_test.go rename to pkg/storage/stores/shipper/indexshipper/storage/util_test.go diff --git a/pkg/storage/stores/indexshipper/table_client.go b/pkg/storage/stores/shipper/indexshipper/table_client.go similarity index 95% rename from pkg/storage/stores/indexshipper/table_client.go rename to pkg/storage/stores/shipper/indexshipper/table_client.go index 068cdd1e36c7a..2d7de63006a61 100644 --- a/pkg/storage/stores/indexshipper/table_client.go +++ b/pkg/storage/stores/shipper/indexshipper/table_client.go @@ -5,8 +5,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" "github.com/grafana/loki/pkg/storage/stores/series/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" ) type tableClient struct { diff --git a/pkg/storage/stores/indexshipper/table_client_test.go b/pkg/storage/stores/shipper/indexshipper/table_client_test.go similarity index 81% rename from pkg/storage/stores/indexshipper/table_client_test.go rename to pkg/storage/stores/shipper/indexshipper/table_client_test.go index f96a0433bbaf7..0b76ab64f99c5 100644 --- a/pkg/storage/stores/indexshipper/table_client_test.go +++ b/pkg/storage/stores/shipper/indexshipper/table_client_test.go @@ -1,4 +1,4 @@ -package indexshipper_test +package indexshipper import ( "bytes" @@ -8,17 +8,14 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" "github.com/grafana/loki/pkg/storage/stores/series/index" ) func TestBoltDBShipperTableClient(t *testing.T) { tempDir := t.TempDir() - cm := storage.NewClientMetrics() - objectClient, err := storage.NewObjectClient("filesystem", storage.Config{FSConfig: local.FSConfig{Directory: tempDir}}, cm) + objectClient, err := local.NewFSObjectClient(local.FSConfig{Directory: tempDir}) require.NoError(t, err) // create a couple of folders with files @@ -36,7 +33,7 @@ func TestBoltDBShipperTableClient(t *testing.T) { } } - tableClient := indexshipper.NewTableClient(objectClient, "index/") + tableClient := NewTableClient(objectClient, "index/") // check list of tables returns all the folders/tables created above checkExpectedTables(t, tableClient, foldersWithFiles) diff --git a/pkg/storage/stores/shipper/testutil/testutil.go b/pkg/storage/stores/shipper/indexshipper/testutil/testutil.go similarity index 100% rename from pkg/storage/stores/shipper/testutil/testutil.go rename to pkg/storage/stores/shipper/indexshipper/testutil/testutil.go diff --git a/pkg/storage/stores/tsdb/bounds.go b/pkg/storage/stores/shipper/indexshipper/tsdb/bounds.go similarity index 100% rename from pkg/storage/stores/tsdb/bounds.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/bounds.go diff --git a/pkg/storage/stores/tsdb/bounds_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/bounds_test.go similarity index 100% rename from pkg/storage/stores/tsdb/bounds_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/bounds_test.go diff --git a/pkg/storage/stores/tsdb/builder.go b/pkg/storage/stores/shipper/indexshipper/tsdb/builder.go similarity index 98% rename from pkg/storage/stores/tsdb/builder.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/builder.go index ec5a64d0a0a1b..ee59debb2e078 100644 --- a/pkg/storage/stores/tsdb/builder.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/builder.go @@ -13,7 +13,7 @@ import ( "github.com/prometheus/prometheus/storage" chunk_util "github.com/grafana/loki/pkg/storage/chunk/client/util" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // Builder is a helper used to create tsdb indices. diff --git a/pkg/storage/stores/tsdb/builder_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/builder_test.go similarity index 97% rename from pkg/storage/stores/tsdb/builder_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/builder_test.go index a3d6aebfd63ff..539cdd57fc5a6 100644 --- a/pkg/storage/stores/tsdb/builder_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/builder_test.go @@ -9,7 +9,7 @@ import ( "github.com/prometheus/common/model" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func Test_Build(t *testing.T) { diff --git a/pkg/storage/stores/tsdb/cached_postings_index.go b/pkg/storage/stores/shipper/indexshipper/tsdb/cached_postings_index.go similarity index 98% rename from pkg/storage/stores/tsdb/cached_postings_index.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/cached_postings_index.go index ec5c99ebda20c..71058a0359efa 100644 --- a/pkg/storage/stores/tsdb/cached_postings_index.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/cached_postings_index.go @@ -17,7 +17,7 @@ import ( "github.com/prometheus/prometheus/storage" "github.com/grafana/loki/pkg/storage/chunk/cache" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) type PostingsReader interface { diff --git a/pkg/storage/stores/tsdb/cached_postings_index_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/cached_postings_index_test.go similarity index 99% rename from pkg/storage/stores/tsdb/cached_postings_index_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/cached_postings_index_test.go index bc43271f764f8..4a7b0f8ee1fec 100644 --- a/pkg/storage/stores/tsdb/cached_postings_index_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/cached_postings_index_test.go @@ -14,7 +14,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/stores/index/stats" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func TestSingleIdxCached(t *testing.T) { diff --git a/pkg/storage/stores/tsdb/compactor.go b/pkg/storage/stores/shipper/indexshipper/tsdb/compactor.go similarity index 94% rename from pkg/storage/stores/tsdb/compactor.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/compactor.go index e6cf35e5cd038..4beb7b0178b42 100644 --- a/pkg/storage/stores/tsdb/compactor.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/compactor.go @@ -17,10 +17,10 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - index_shipper "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) const readDBsConcurrency = 50 @@ -53,7 +53,7 @@ func (i indexProcessor) OpenCompactedIndexFile(ctx context.Context, path, tableN } builder := NewBuilder(indexFormat) - err = indexFile.(*TSDBFile).Index.(*TSDBIndex).ForSeries(ctx, nil, 0, math.MaxInt64, func(lbls labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) { + err = indexFile.(*TSDBFile).Index.(*TSDBIndex).ForSeries(ctx, nil, 0, math.MaxInt64, func(lbls labels.Labels, fp model.Fingerprint, chks []tsdbindex.ChunkMeta) { builder.AddSeries(lbls.Copy(), fp, chks) }, labels.MustNewMatcher(labels.MatchEqual, "", "")) if err != nil { @@ -212,7 +212,7 @@ func setupBuilder(ctx context.Context, indexType int, userID string, sourceIndex // add users index from multi-tenant indexes to the builder for _, idx := range multiTenantIndexes { - err := idx.(*TSDBFile).Index.(*TSDBIndex).ForSeries(ctx, nil, 0, math.MaxInt64, func(lbls labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) { + err := idx.(*TSDBFile).Index.(*TSDBIndex).ForSeries(ctx, nil, 0, math.MaxInt64, func(lbls labels.Labels, fp model.Fingerprint, chks []tsdbindex.ChunkMeta) { builder.AddSeries(withoutTenantLabel(lbls.Copy()), fp, chks) }, withTenantLabelMatcher(userID, []*labels.Matcher{})...) if err != nil { @@ -244,7 +244,7 @@ func setupBuilder(ctx context.Context, indexType int, userID string, sourceIndex } }() - err = indexFile.(*TSDBFile).Index.(*TSDBIndex).ForSeries(ctx, nil, 0, math.MaxInt64, func(lbls labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) { + err = indexFile.(*TSDBFile).Index.(*TSDBIndex).ForSeries(ctx, nil, 0, math.MaxInt64, func(lbls labels.Labels, fp model.Fingerprint, chks []tsdbindex.ChunkMeta) { builder.AddSeries(lbls.Copy(), fp, chks) }, labels.MustNewMatcher(labels.MatchEqual, "", "")) if err != nil { @@ -267,7 +267,7 @@ type compactedIndex struct { periodConfig config.PeriodConfig indexChunks []chunk.Chunk - deleteChunks map[string][]index.ChunkMeta + deleteChunks map[string][]tsdbindex.ChunkMeta seriesToCleanup map[string]struct{} } @@ -279,7 +279,7 @@ func newCompactedIndex(ctx context.Context, tableName, userID, workingDir string workingDir: workingDir, periodConfig: periodConfig, tableInterval: retention.ExtractIntervalFromTableName(tableName), - deleteChunks: map[string][]index.ChunkMeta{}, + deleteChunks: map[string][]tsdbindex.ChunkMeta{}, seriesToCleanup: map[string]struct{}{}, } } @@ -355,7 +355,7 @@ func (c *compactedIndex) Cleanup() {} // ToIndexFile creates an indexFile from the chunksmetas stored in the builder. // Before building the index, it takes care of the lined up updates i.e deletes and adding of new chunks. -func (c *compactedIndex) ToIndexFile() (index_shipper.Index, error) { +func (c *compactedIndex) ToIndexFile() (shipperindex.Index, error) { for seriesID, chks := range c.deleteChunks { for _, chk := range chks { chunkFound, err := c.builder.DropChunk(seriesID, chk) @@ -376,7 +376,7 @@ func (c *compactedIndex) ToIndexFile() (index_shipper.Index, error) { ls := b.Labels() approxKB := math.Round(float64(chk.Data.UncompressedSize()) / float64(1<<10)) - err := c.builder.InsertChunk(ls.String(), index.ChunkMeta{ + err := c.builder.InsertChunk(ls.String(), tsdbindex.ChunkMeta{ Checksum: chk.Checksum, MinTime: int64(chk.From), MaxTime: int64(chk.Through), diff --git a/pkg/storage/stores/tsdb/compactor_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/compactor_test.go similarity index 98% rename from pkg/storage/stores/tsdb/compactor_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/compactor_test.go index 226eeb00365ae..a34df17b47c80 100644 --- a/pkg/storage/stores/tsdb/compactor_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/compactor_test.go @@ -23,10 +23,10 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/tsdb/head.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head.go similarity index 99% rename from pkg/storage/stores/tsdb/head.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/head.go index f2101028d70b8..4c841b3741462 100644 --- a/pkg/storage/stores/tsdb/head.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head.go @@ -22,7 +22,7 @@ import ( "github.com/prometheus/prometheus/storage" "go.uber.org/atomic" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) /* diff --git a/pkg/storage/stores/tsdb/head_manager.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager.go similarity index 99% rename from pkg/storage/stores/tsdb/head_manager.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/head_manager.go index c1932c27fdc82..47f102f17c77f 100644 --- a/pkg/storage/stores/tsdb/head_manager.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager.go @@ -23,7 +23,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client/util" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util/wal" ) diff --git a/pkg/storage/stores/tsdb/head_manager_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go similarity index 99% rename from pkg/storage/stores/tsdb/head_manager_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go index ce15a2042c26c..1b488e4db81c9 100644 --- a/pkg/storage/stores/tsdb/head_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head_manager_test.go @@ -22,8 +22,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/validation" ) diff --git a/pkg/storage/stores/tsdb/head_read.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head_read.go similarity index 98% rename from pkg/storage/stores/tsdb/head_read.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/head_read.go index e17363ff9aa35..4ffc8ae2e9a96 100644 --- a/pkg/storage/stores/tsdb/head_read.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head_read.go @@ -21,7 +21,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // Index returns an IndexReader against the block. diff --git a/pkg/storage/stores/tsdb/head_wal.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head_wal.go similarity index 98% rename from pkg/storage/stores/tsdb/head_wal.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/head_wal.go index 6a31e4105af5b..7ac700bf1cc1a 100644 --- a/pkg/storage/stores/tsdb/head_wal.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head_wal.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/prometheus/tsdb/record" "github.com/prometheus/prometheus/tsdb/wlog" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util/encoding" ) diff --git a/pkg/storage/stores/tsdb/head_wal_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/head_wal_test.go similarity index 96% rename from pkg/storage/stores/tsdb/head_wal_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/head_wal_test.go index 3a56304172d77..28255a4a7ec88 100644 --- a/pkg/storage/stores/tsdb/head_wal_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/head_wal_test.go @@ -9,7 +9,7 @@ import ( "github.com/prometheus/prometheus/tsdb/record" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func Test_Encoding_Series(t *testing.T) { diff --git a/pkg/storage/stores/tsdb/identifier.go b/pkg/storage/stores/shipper/indexshipper/tsdb/identifier.go similarity index 100% rename from pkg/storage/stores/tsdb/identifier.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/identifier.go diff --git a/pkg/storage/stores/tsdb/identifier_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/identifier_test.go similarity index 100% rename from pkg/storage/stores/tsdb/identifier_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/identifier_test.go diff --git a/pkg/storage/stores/tsdb/index.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index.go similarity index 96% rename from pkg/storage/stores/tsdb/index.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index.go index 74e6259143521..97a5bf8a60d8a 100644 --- a/pkg/storage/stores/tsdb/index.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/index.go @@ -8,8 +8,8 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) type Series struct { diff --git a/pkg/storage/stores/tsdb/index/chunk.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/chunk.go similarity index 100% rename from pkg/storage/stores/tsdb/index/chunk.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/chunk.go diff --git a/pkg/storage/stores/tsdb/index/chunk_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/chunk_test.go similarity index 100% rename from pkg/storage/stores/tsdb/index/chunk_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/chunk_test.go diff --git a/pkg/storage/stores/tsdb/index/fingerprint.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/fingerprint.go similarity index 100% rename from pkg/storage/stores/tsdb/index/fingerprint.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/fingerprint.go diff --git a/pkg/storage/stores/tsdb/index/fingerprint_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/fingerprint_test.go similarity index 100% rename from pkg/storage/stores/tsdb/index/fingerprint_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/fingerprint_test.go diff --git a/pkg/storage/stores/tsdb/index/index.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/index.go similarity index 100% rename from pkg/storage/stores/tsdb/index/index.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/index.go diff --git a/pkg/storage/stores/tsdb/index/index_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/index_test.go similarity index 100% rename from pkg/storage/stores/tsdb/index/index_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/index_test.go diff --git a/pkg/storage/stores/tsdb/index/pool.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/pool.go similarity index 100% rename from pkg/storage/stores/tsdb/index/pool.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/pool.go diff --git a/pkg/storage/stores/tsdb/index/postings.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/postings.go similarity index 100% rename from pkg/storage/stores/tsdb/index/postings.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/postings.go diff --git a/pkg/storage/stores/tsdb/index/postings_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/postings_test.go similarity index 100% rename from pkg/storage/stores/tsdb/index/postings_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/postings_test.go diff --git a/pkg/storage/stores/tsdb/index/postingsstats.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/postingsstats.go similarity index 100% rename from pkg/storage/stores/tsdb/index/postingsstats.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/postingsstats.go diff --git a/pkg/storage/stores/tsdb/index/postingsstats_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/postingsstats_test.go similarity index 100% rename from pkg/storage/stores/tsdb/index/postingsstats_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/postingsstats_test.go diff --git a/pkg/storage/stores/tsdb/index/shard.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/shard.go similarity index 100% rename from pkg/storage/stores/tsdb/index/shard.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/shard.go diff --git a/pkg/storage/stores/tsdb/index/shard_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/shard_test.go similarity index 100% rename from pkg/storage/stores/tsdb/index/shard_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/shard_test.go diff --git a/pkg/storage/stores/tsdb/index/test_utils.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index/test_utils.go similarity index 100% rename from pkg/storage/stores/tsdb/index/test_utils.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index/test_utils.go diff --git a/pkg/storage/stores/tsdb/index_client.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index_client.go similarity index 99% rename from pkg/storage/stores/tsdb/index_client.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index_client.go index 7b882e46f7715..9c8dde24bb132 100644 --- a/pkg/storage/stores/tsdb/index_client.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/index_client.go @@ -16,7 +16,7 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/index/stats" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util" ) diff --git a/pkg/storage/stores/tsdb/index_client_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index_client_test.go similarity index 97% rename from pkg/storage/stores/tsdb/index_client_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index_client_test.go index d7ef5ea6f6205..d9f2829ec4af5 100644 --- a/pkg/storage/stores/tsdb/index_client_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/index_client_test.go @@ -7,27 +7,25 @@ import ( "testing" "time" - "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" - - "github.com/grafana/loki/pkg/logproto" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" + "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/storage/config" - index_shipper "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" ) type mockIndexShipperIndexIterator struct { tables map[string][]*TSDBFile } -func (m mockIndexShipperIndexIterator) ForEachConcurrent(ctx context.Context, tableName, userID string, callback index_shipper.ForEachIndexCallback) error { +func (m mockIndexShipperIndexIterator) ForEachConcurrent(ctx context.Context, tableName, userID string, callback shipperindex.ForEachIndexCallback) error { return m.ForEach(ctx, tableName, userID, callback) } -func (m mockIndexShipperIndexIterator) ForEach(_ context.Context, tableName, _ string, callback index_shipper.ForEachIndexCallback) error { +func (m mockIndexShipperIndexIterator) ForEach(_ context.Context, tableName, _ string, callback shipperindex.ForEachIndexCallback) error { indexes := m.tables[tableName] for _, idx := range indexes { if err := callback(false, idx); err != nil { diff --git a/pkg/storage/stores/tsdb/index_shipper_querier.go b/pkg/storage/stores/shipper/indexshipper/tsdb/index_shipper_querier.go similarity index 86% rename from pkg/storage/stores/tsdb/index_shipper_querier.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/index_shipper_querier.go index e72f1c9053a54..1a89df0c26d97 100644 --- a/pkg/storage/stores/tsdb/index_shipper_querier.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/index_shipper_querier.go @@ -12,12 +12,12 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/config" - shipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) type indexShipperIterator interface { - ForEachConcurrent(ctx context.Context, tableName, userID string, callback shipper_index.ForEachIndexCallback) error + ForEachConcurrent(ctx context.Context, tableName, userID string, callback shipperindex.ForEachIndexCallback) error } // indexShipperQuerier is used for querying index from the shipper. @@ -42,7 +42,7 @@ func (i *indexShipperQuerier) indices(ctx context.Context, from, through model.T // Ensure we query both per tenant and multitenant TSDBs idxBuckets := indexBuckets(from, through, []config.TableRange{i.tableRange}) for _, bkt := range idxBuckets { - if err := i.shipper.ForEachConcurrent(ctx, bkt.prefix, user, func(multitenant bool, idx shipper_index.Index) error { + if err := i.shipper.ForEachConcurrent(ctx, bkt.prefix, user, func(multitenant bool, idx shipperindex.Index) error { impl, ok := idx.(Index) if !ok { return fmt.Errorf("unexpected shipper index type: %T", idx) @@ -84,7 +84,7 @@ func (i *indexShipperQuerier) Close() error { return nil } -func (i *indexShipperQuerier) GetChunkRefs(ctx context.Context, userID string, from, through model.Time, res []ChunkRef, shard *index.ShardAnnotation, matchers ...*labels.Matcher) ([]ChunkRef, error) { +func (i *indexShipperQuerier) GetChunkRefs(ctx context.Context, userID string, from, through model.Time, res []ChunkRef, shard *tsdbindex.ShardAnnotation, matchers ...*labels.Matcher) ([]ChunkRef, error) { idx, err := i.indices(ctx, from, through, userID) if err != nil { return nil, err @@ -92,7 +92,7 @@ func (i *indexShipperQuerier) GetChunkRefs(ctx context.Context, userID string, f return idx.GetChunkRefs(ctx, userID, from, through, res, shard, matchers...) } -func (i *indexShipperQuerier) Series(ctx context.Context, userID string, from, through model.Time, res []Series, shard *index.ShardAnnotation, matchers ...*labels.Matcher) ([]Series, error) { +func (i *indexShipperQuerier) Series(ctx context.Context, userID string, from, through model.Time, res []Series, shard *tsdbindex.ShardAnnotation, matchers ...*labels.Matcher) ([]Series, error) { idx, err := i.indices(ctx, from, through, userID) if err != nil { return nil, err @@ -116,7 +116,7 @@ func (i *indexShipperQuerier) LabelValues(ctx context.Context, userID string, fr return idx.LabelValues(ctx, userID, from, through, name, matchers...) } -func (i *indexShipperQuerier) Stats(ctx context.Context, userID string, from, through model.Time, acc IndexStatsAccumulator, shard *index.ShardAnnotation, shouldIncludeChunk shouldIncludeChunk, matchers ...*labels.Matcher) error { +func (i *indexShipperQuerier) Stats(ctx context.Context, userID string, from, through model.Time, acc IndexStatsAccumulator, shard *tsdbindex.ShardAnnotation, shouldIncludeChunk shouldIncludeChunk, matchers ...*labels.Matcher) error { idx, err := i.indices(ctx, from, through, userID) if err != nil { return err @@ -125,7 +125,7 @@ func (i *indexShipperQuerier) Stats(ctx context.Context, userID string, from, th return idx.Stats(ctx, userID, from, through, acc, shard, shouldIncludeChunk, matchers...) } -func (i *indexShipperQuerier) Volume(ctx context.Context, userID string, from, through model.Time, acc VolumeAccumulator, shard *index.ShardAnnotation, shouldIncludeChunk shouldIncludeChunk, targetLabels []string, aggregateBy string, matchers ...*labels.Matcher) error { +func (i *indexShipperQuerier) Volume(ctx context.Context, userID string, from, through model.Time, acc VolumeAccumulator, shard *tsdbindex.ShardAnnotation, shouldIncludeChunk shouldIncludeChunk, targetLabels []string, aggregateBy string, matchers ...*labels.Matcher) error { idx, err := i.indices(ctx, from, through, userID) if err != nil { return err diff --git a/pkg/storage/stores/tsdb/lazy_index.go b/pkg/storage/stores/shipper/indexshipper/tsdb/lazy_index.go similarity index 97% rename from pkg/storage/stores/tsdb/lazy_index.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/lazy_index.go index e6906bef59f44..b9877a6ae1ab3 100644 --- a/pkg/storage/stores/tsdb/lazy_index.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/lazy_index.go @@ -7,7 +7,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // Index adapter for a function which returns an index when queried. diff --git a/pkg/storage/stores/tsdb/manager.go b/pkg/storage/stores/shipper/indexshipper/tsdb/manager.go similarity index 92% rename from pkg/storage/stores/tsdb/manager.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/manager.go index f4ea7978d9613..3b9c10fc3fed2 100644 --- a/pkg/storage/stores/tsdb/manager.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/manager.go @@ -16,8 +16,8 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -58,7 +58,7 @@ func NewTSDBManager( name, nodeName, dir string, - shipper indexshipper.IndexShipper, + indexShipper indexshipper.IndexShipper, tableRange config.TableRange, schemaCfg config.SchemaConfig, logger log.Logger, @@ -72,7 +72,7 @@ func NewTSDBManager( metrics: metrics, tableRange: tableRange, schemaCfg: schemaCfg, - shipper: shipper, + shipper: indexShipper, } } @@ -156,7 +156,7 @@ type chunkInfo struct { tsdbFormat int } -func (m *tsdbManager) buildFromHead(heads *tenantHeads, shipper indexshipper.IndexShipper, tableRanges []config.TableRange) (err error) { +func (m *tsdbManager) buildFromHead(heads *tenantHeads, indexShipper indexshipper.IndexShipper, tableRanges []config.TableRange) (err error) { periods := make(map[string]*Builder) if err := heads.forAll(func(user string, ls labels.Labels, fp uint64, chks index.ChunkMetas) error { @@ -235,7 +235,7 @@ func (m *tsdbManager) buildFromHead(heads *tenantHeads, shipper indexshipper.Ind return err } - if err := shipper.AddIndex(p, "", loaded); err != nil { + if err := indexShipper.AddIndex(p, "", loaded); err != nil { return err } } @@ -270,8 +270,8 @@ func (m *tsdbManager) BuildFromWALs(t time.Time, ids []WALIdentifier, legacy boo }() var ( - tableRanges = []config.TableRange{m.tableRange} - shipper = m.shipper + tableRanges = []config.TableRange{m.tableRange} + indexShipper = m.shipper ) if legacy { @@ -280,7 +280,7 @@ func (m *tsdbManager) BuildFromWALs(t time.Time, ids []WALIdentifier, legacy boo // do not ship legacy WAL files. // TSDBs built from these WAL files would get loaded on starting tsdbManager - shipper = indexshipper.Noop{} + indexShipper = indexshipper.Noop{} } level.Debug(m.log).Log("msg", "recovering tenant heads") @@ -290,7 +290,7 @@ func (m *tsdbManager) BuildFromWALs(t time.Time, ids []WALIdentifier, legacy boo return errors.Wrap(err, "building TSDB from WALs") } - err := m.buildFromHead(tmp, shipper, tableRanges) + err := m.buildFromHead(tmp, indexShipper, tableRanges) if err != nil { return err } diff --git a/pkg/storage/stores/tsdb/multi_file_index.go b/pkg/storage/stores/shipper/indexshipper/tsdb/multi_file_index.go similarity index 99% rename from pkg/storage/stores/tsdb/multi_file_index.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/multi_file_index.go index 5837db7cc71ba..41f1f1a834414 100644 --- a/pkg/storage/stores/tsdb/multi_file_index.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/multi_file_index.go @@ -10,7 +10,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) type MultiIndex struct { diff --git a/pkg/storage/stores/tsdb/multi_file_index_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/multi_file_index_test.go similarity index 98% rename from pkg/storage/stores/tsdb/multi_file_index_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/multi_file_index_test.go index e82d158bd6dba..829c942e7b6d5 100644 --- a/pkg/storage/stores/tsdb/multi_file_index_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/multi_file_index_test.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func TestMultiIndex(t *testing.T) { diff --git a/pkg/storage/stores/tsdb/multitenant.go b/pkg/storage/stores/shipper/indexshipper/tsdb/multitenant.go similarity index 97% rename from pkg/storage/stores/tsdb/multitenant.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/multitenant.go index f24d6ecdfad2d..9dda0886dbf40 100644 --- a/pkg/storage/stores/tsdb/multitenant.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/multitenant.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // TenantLabel is part of the reserved label namespace (__ prefix) diff --git a/pkg/storage/stores/tsdb/pool.go b/pkg/storage/stores/shipper/indexshipper/tsdb/pool.go similarity index 90% rename from pkg/storage/stores/tsdb/pool.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/pool.go index 0b02f5c11be60..f5a009d37bc12 100644 --- a/pkg/storage/stores/tsdb/pool.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/pool.go @@ -3,7 +3,7 @@ package tsdb import ( "sync" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) var ( diff --git a/pkg/storage/stores/tsdb/postings_codec.go b/pkg/storage/stores/shipper/indexshipper/tsdb/postings_codec.go similarity index 97% rename from pkg/storage/stores/tsdb/postings_codec.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/postings_codec.go index a6517352213ee..629024c67728d 100644 --- a/pkg/storage/stores/tsdb/postings_codec.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/postings_codec.go @@ -8,7 +8,7 @@ import ( "github.com/prometheus/prometheus/storage" promEncoding "github.com/prometheus/prometheus/tsdb/encoding" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util/encoding" ) diff --git a/pkg/storage/stores/tsdb/postings_codec_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/postings_codec_test.go similarity index 98% rename from pkg/storage/stores/tsdb/postings_codec_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/postings_codec_test.go index 41f8c8efd107a..3bc0510cdf741 100644 --- a/pkg/storage/stores/tsdb/postings_codec_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/postings_codec_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) const ( diff --git a/pkg/storage/stores/tsdb/querier.go b/pkg/storage/stores/shipper/indexshipper/tsdb/querier.go similarity index 99% rename from pkg/storage/stores/tsdb/querier.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/querier.go index a9ae060263575..10498bdf94e3f 100644 --- a/pkg/storage/stores/tsdb/querier.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/querier.go @@ -22,7 +22,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) // Bitmap used by func isRegexMetaCharacter to check whether a character needs to be escaped. diff --git a/pkg/storage/stores/tsdb/querier_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/querier_test.go similarity index 97% rename from pkg/storage/stores/tsdb/querier_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/querier_test.go index 09d978c27e7df..a0873faeb6a20 100644 --- a/pkg/storage/stores/tsdb/querier_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/querier_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logql/syntax" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func mustParseLabels(s string) labels.Labels { diff --git a/pkg/storage/stores/tsdb/single_file_index.go b/pkg/storage/stores/shipper/indexshipper/tsdb/single_file_index.go similarity index 97% rename from pkg/storage/stores/tsdb/single_file_index.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/single_file_index.go index b913e44c530a4..e6c9866b0227e 100644 --- a/pkg/storage/stores/tsdb/single_file_index.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/single_file_index.go @@ -18,8 +18,8 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" - index_shipper "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -33,7 +33,7 @@ type IndexOpts struct { PostingsCache cache.Cache } -func OpenShippableTSDB(p string, opts IndexOpts) (index_shipper.Index, error) { +func OpenShippableTSDB(p string, opts IndexOpts) (shipperindex.Index, error) { id, err := identifierFromPath(p) if err != nil { return nil, err @@ -42,7 +42,7 @@ func OpenShippableTSDB(p string, opts IndexOpts) (index_shipper.Index, error) { return NewShippableTSDBFile(id, opts) } -func RebuildWithVersion(ctx context.Context, path string, desiredVer int) (index_shipper.Index, error) { +func RebuildWithVersion(ctx context.Context, path string, desiredVer int) (shipperindex.Index, error) { opts := IndexOpts{} indexFile, err := OpenShippableTSDB(path, opts) if err != nil { diff --git a/pkg/storage/stores/tsdb/single_file_index_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/single_file_index_test.go similarity index 99% rename from pkg/storage/stores/tsdb/single_file_index_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/single_file_index_test.go index 7df713eece97a..ffc5a0871fed4 100644 --- a/pkg/storage/stores/tsdb/single_file_index_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/single_file_index_test.go @@ -7,17 +7,16 @@ import ( "testing" "time" - "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/storage/chunk" - "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" - "github.com/grafana/loki/pkg/storage/stores/index/stats" - "github.com/go-kit/log" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/storage/chunk" + "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" + "github.com/grafana/loki/pkg/storage/stores/index/stats" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func TestSingleIdx(t *testing.T) { diff --git a/pkg/storage/stores/tsdb/store.go b/pkg/storage/stores/shipper/indexshipper/tsdb/store.go similarity index 90% rename from pkg/storage/stores/tsdb/store.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/store.go index 515d0f8162d46..5bd21a595e344 100644 --- a/pkg/storage/stores/tsdb/store.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/store.go @@ -19,14 +19,14 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/fetcher" "github.com/grafana/loki/pkg/storage/config" "github.com/grafana/loki/pkg/storage/stores/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/downloads" - indexshipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - tsdb_index "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/downloads" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) type IndexWriter interface { - Append(userID string, ls labels.Labels, fprint uint64, chks tsdb_index.ChunkMetas) error + Append(userID string, ls labels.Labels, fprint uint64, chks tsdbindex.ChunkMetas) error } type store struct { @@ -74,7 +74,7 @@ func (s *store) init(name string, indexCfg IndexCfg, schemaCfg config.SchemaConf sharedCache = idxCache } - openFn := func(p string) (indexshipper_index.Index, error) { + openFn := func(p string) (shipperindex.Index, error) { return OpenShippableTSDB(p, IndexOpts{PostingsCache: sharedCache}) } @@ -164,7 +164,7 @@ func (s *store) Stop() { func (s *store) IndexChunk(ctx context.Context, from model.Time, through model.Time, chk chunk.Chunk) error { // Always write the index to benefit durability via replication factor. approxKB := math.Round(float64(chk.Data.UncompressedSize()) / float64(1<<10)) - metas := tsdb_index.ChunkMetas{ + metas := tsdbindex.ChunkMetas{ { Checksum: chk.ChunkRef.Checksum, MinTime: int64(chk.ChunkRef.From), @@ -181,6 +181,6 @@ func (s *store) IndexChunk(ctx context.Context, from model.Time, through model.T type failingIndexWriter struct{} -func (f failingIndexWriter) Append(_ string, _ labels.Labels, _ uint64, _ tsdb_index.ChunkMetas) error { +func (f failingIndexWriter) Append(_ string, _ labels.Labels, _ uint64, _ tsdbindex.ChunkMetas) error { return fmt.Errorf("index writer is not initialized due to tsdb store being initialized in read-only mode") } diff --git a/pkg/storage/stores/tsdb/testdata/20kseries.json b/pkg/storage/stores/shipper/indexshipper/tsdb/testdata/20kseries.json similarity index 100% rename from pkg/storage/stores/tsdb/testdata/20kseries.json rename to pkg/storage/stores/shipper/indexshipper/tsdb/testdata/20kseries.json diff --git a/pkg/storage/stores/tsdb/testutil/objstore.go b/pkg/storage/stores/shipper/indexshipper/tsdb/testutil/objstore.go similarity index 100% rename from pkg/storage/stores/tsdb/testutil/objstore.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/testutil/objstore.go diff --git a/pkg/storage/stores/tsdb/util_test.go b/pkg/storage/stores/shipper/indexshipper/tsdb/util_test.go similarity index 91% rename from pkg/storage/stores/tsdb/util_test.go rename to pkg/storage/stores/shipper/indexshipper/tsdb/util_test.go index dab02b1d2ce27..13e2c6f54e0ac 100644 --- a/pkg/storage/stores/tsdb/util_test.go +++ b/pkg/storage/stores/shipper/indexshipper/tsdb/util_test.go @@ -9,7 +9,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) type LoadableSeries struct { diff --git a/pkg/storage/stores/indexshipper/uploads/index_set.go b/pkg/storage/stores/shipper/indexshipper/uploads/index_set.go similarity index 97% rename from pkg/storage/stores/indexshipper/uploads/index_set.go rename to pkg/storage/stores/shipper/indexshipper/uploads/index_set.go index 612124ba5c2c3..7cbe4b21b4d86 100644 --- a/pkg/storage/stores/indexshipper/uploads/index_set.go +++ b/pkg/storage/stores/shipper/indexshipper/uploads/index_set.go @@ -12,8 +12,8 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/loki/pkg/chunkenc" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/uploads/index_set_test.go b/pkg/storage/stores/shipper/indexshipper/uploads/index_set_test.go similarity index 95% rename from pkg/storage/stores/indexshipper/uploads/index_set_test.go rename to pkg/storage/stores/shipper/indexshipper/uploads/index_set_test.go index 15da352f42ff0..d6d772734195c 100644 --- a/pkg/storage/stores/indexshipper/uploads/index_set_test.go +++ b/pkg/storage/stores/shipper/indexshipper/uploads/index_set_test.go @@ -10,9 +10,9 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/shipper/testutil" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/testutil" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/uploads/metrics.go b/pkg/storage/stores/shipper/indexshipper/uploads/metrics.go similarity index 100% rename from pkg/storage/stores/indexshipper/uploads/metrics.go rename to pkg/storage/stores/shipper/indexshipper/uploads/metrics.go diff --git a/pkg/storage/stores/indexshipper/uploads/table.go b/pkg/storage/stores/shipper/indexshipper/uploads/table.go similarity index 96% rename from pkg/storage/stores/indexshipper/uploads/table.go rename to pkg/storage/stores/shipper/indexshipper/uploads/table.go index 74bda5c281b55..2e092ce3727a2 100644 --- a/pkg/storage/stores/indexshipper/uploads/table.go +++ b/pkg/storage/stores/shipper/indexshipper/uploads/table.go @@ -7,8 +7,8 @@ import ( "github.com/go-kit/log" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" util_log "github.com/grafana/loki/pkg/util/log" ) diff --git a/pkg/storage/stores/indexshipper/uploads/table_manager.go b/pkg/storage/stores/shipper/indexshipper/uploads/table_manager.go similarity index 95% rename from pkg/storage/stores/indexshipper/uploads/table_manager.go rename to pkg/storage/stores/shipper/indexshipper/uploads/table_manager.go index 32960251a4388..6d9aa9e35d487 100644 --- a/pkg/storage/stores/indexshipper/uploads/table_manager.go +++ b/pkg/storage/stores/shipper/indexshipper/uploads/table_manager.go @@ -9,8 +9,8 @@ import ( "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" ) type Config struct { diff --git a/pkg/storage/stores/indexshipper/uploads/table_manager_test.go b/pkg/storage/stores/shipper/indexshipper/uploads/table_manager_test.go similarity index 93% rename from pkg/storage/stores/indexshipper/uploads/table_manager_test.go rename to pkg/storage/stores/shipper/indexshipper/uploads/table_manager_test.go index c2c8c15a41450..7013ff389c782 100644 --- a/pkg/storage/stores/indexshipper/uploads/table_manager_test.go +++ b/pkg/storage/stores/shipper/indexshipper/uploads/table_manager_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" ) const objectsStorageDirName = "objects" diff --git a/pkg/storage/stores/indexshipper/uploads/table_test.go b/pkg/storage/stores/shipper/indexshipper/uploads/table_test.go similarity index 94% rename from pkg/storage/stores/indexshipper/uploads/table_test.go rename to pkg/storage/stores/shipper/indexshipper/uploads/table_test.go index a6e284b609687..f67b3d3571c28 100644 --- a/pkg/storage/stores/indexshipper/uploads/table_test.go +++ b/pkg/storage/stores/shipper/indexshipper/uploads/table_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" ) const ( diff --git a/pkg/storage/stores/indexshipper/uploads/testutil.go b/pkg/storage/stores/shipper/indexshipper/uploads/testutil.go similarity index 100% rename from pkg/storage/stores/indexshipper/uploads/testutil.go rename to pkg/storage/stores/shipper/indexshipper/uploads/testutil.go diff --git a/pkg/storage/stores/shipper/util/queries.go b/pkg/storage/stores/shipper/indexshipper/util/queries.go similarity index 100% rename from pkg/storage/stores/shipper/util/queries.go rename to pkg/storage/stores/shipper/indexshipper/util/queries.go diff --git a/pkg/storage/stores/shipper/util/queries_test.go b/pkg/storage/stores/shipper/indexshipper/util/queries_test.go similarity index 100% rename from pkg/storage/stores/shipper/util/queries_test.go rename to pkg/storage/stores/shipper/indexshipper/util/queries_test.go diff --git a/pkg/storage/stores/shipper/util/util.go b/pkg/storage/stores/shipper/indexshipper/util/util.go similarity index 83% rename from pkg/storage/stores/shipper/util/util.go rename to pkg/storage/stores/shipper/indexshipper/util/util.go index 6f26c8da262da..9150ab34a0cbc 100644 --- a/pkg/storage/stores/shipper/util/util.go +++ b/pkg/storage/stores/shipper/indexshipper/util/util.go @@ -10,7 +10,6 @@ import ( "go.etcd.io/bbolt" "github.com/grafana/loki/pkg/storage/chunk/client/local" - "github.com/grafana/loki/pkg/storage/stores/series/index" ) const maxStackSize = 8 * 1024 @@ -64,23 +63,23 @@ func safeOpenBoltDbFile(path string, ret chan *result) { res.err = err } -func QueryKey(q index.Query) string { - ret := q.TableName + sep + q.HashValue +// func QueryKey(q index.Query) string { +// ret := q.TableName + sep + q.HashValue - if len(q.RangeValuePrefix) != 0 { - ret += sep + string(q.RangeValuePrefix) - } +// if len(q.RangeValuePrefix) != 0 { +// ret += sep + string(q.RangeValuePrefix) +// } - if len(q.RangeValueStart) != 0 { - ret += sep + string(q.RangeValueStart) - } +// if len(q.RangeValueStart) != 0 { +// ret += sep + string(q.RangeValueStart) +// } - if len(q.ValueEqual) != 0 { - ret += sep + string(q.ValueEqual) - } +// if len(q.ValueEqual) != 0 { +// ret += sep + string(q.ValueEqual) +// } - return ret -} +// return ret +// } func GetUnsafeBytes(s string) []byte { return *((*[]byte)(unsafe.Pointer(&s))) diff --git a/pkg/storage/stores/shipper/metrics.go b/pkg/storage/stores/shipper/metrics.go deleted file mode 100644 index 63f84fd38de17..0000000000000 --- a/pkg/storage/stores/shipper/metrics.go +++ /dev/null @@ -1,23 +0,0 @@ -package shipper - -import ( - "github.com/grafana/dskit/instrument" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -type metrics struct { - // duration in seconds spent in serving request on index managed by BoltDB Shipper - requestDurationSeconds *prometheus.HistogramVec -} - -func newMetrics(r prometheus.Registerer) *metrics { - return &metrics{ - requestDurationSeconds: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "loki_boltdb_shipper", - Name: "request_duration_seconds", - Help: "Time (in seconds) spent serving requests when using boltdb shipper", - Buckets: instrument.DefBuckets, - }, []string{"operation", "status_code"}), - } -} diff --git a/pkg/util/limiter/combined_limits.go b/pkg/util/limiter/combined_limits.go index c835d69aa8fe4..700460ca8e86b 100644 --- a/pkg/util/limiter/combined_limits.go +++ b/pkg/util/limiter/combined_limits.go @@ -8,8 +8,8 @@ import ( "github.com/grafana/loki/pkg/ruler" "github.com/grafana/loki/pkg/scheduler" "github.com/grafana/loki/pkg/storage" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" ) type CombinedLimits interface { diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index cbcd6c2ffb2a7..6730fea5fbb90 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -23,7 +23,7 @@ import ( "github.com/grafana/loki/pkg/logql/syntax" ruler_config "github.com/grafana/loki/pkg/ruler/config" "github.com/grafana/loki/pkg/ruler/util" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode" "github.com/grafana/loki/pkg/util/flagext" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/util/validation" diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go index 89e3516bf9ba2..22a0157a39e2d 100644 --- a/pkg/validation/limits_test.go +++ b/pkg/validation/limits_test.go @@ -7,13 +7,12 @@ import ( "time" "github.com/pkg/errors" - - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/deletionmode" - "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/deletionmode" ) func TestLimitsTagsYamlMatchJson(t *testing.T) { diff --git a/tools/doc-generator/parse/root_blocks.go b/tools/doc-generator/parse/root_blocks.go index de6d99113a201..e2dedd316c454 100644 --- a/tools/doc-generator/parse/root_blocks.go +++ b/tools/doc-generator/parse/root_blocks.go @@ -35,9 +35,9 @@ import ( "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/chunk/client/openstack" storage_config "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor" "github.com/grafana/loki/pkg/storage/stores/series/index" - "github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/indexgateway" "github.com/grafana/loki/pkg/tracing" "github.com/grafana/loki/pkg/validation" ) diff --git a/tools/tsdb/bloom-tester/concurrent.go b/tools/tsdb/bloom-tester/concurrent.go index 5c097af399e80..9f7fd8c1a0bbf 100644 --- a/tools/tsdb/bloom-tester/concurrent.go +++ b/tools/tsdb/bloom-tester/concurrent.go @@ -1,7 +1,7 @@ package main import ( - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" ) diff --git a/tools/tsdb/bloom-tester/lib.go b/tools/tsdb/bloom-tester/lib.go index b9af2534211cb..3ea4d2d674c61 100644 --- a/tools/tsdb/bloom-tester/lib.go +++ b/tools/tsdb/bloom-tester/lib.go @@ -9,12 +9,12 @@ import ( "time" "github.com/go-kit/log/level" + "github.com/grafana/dskit/services" "github.com/owen-d/BoomFilters/boom" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" - "github.com/grafana/dskit/services" "github.com/grafana/loki/pkg/chunkenc" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/log" @@ -22,10 +22,10 @@ import ( "github.com/grafana/loki/pkg/storage/chunk" "github.com/grafana/loki/pkg/storage/chunk/client" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - indexshipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/tsdb" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/tools/tsdb/helpers" ) @@ -45,11 +45,11 @@ func execute() { tableRanges := helpers.GetIndexStoreTableRanges(config.TSDBType, conf.SchemaConfig.Configs) - openFn := func(p string) (indexshipper_index.Index, error) { + openFn := func(p string) (shipperindex.Index, error) { return tsdb.OpenShippableTSDB(p, tsdb.IndexOpts{}) } - shipper, err := indexshipper.NewIndexShipper( + indexShipper, err := indexshipper.NewIndexShipper( conf.StorageConfig.TSDBShipperConfig.Config, objectClient, overrides, @@ -74,7 +74,7 @@ func execute() { helpers.ExitErr("waiting for service to start", err) level.Info(util_log.Logger).Log("msg", "server started") - err = analyze(metrics, sampler, shipper, chunkClient, tableName, tenants) + err = analyze(metrics, sampler, indexShipper, chunkClient, tableName, tenants) helpers.ExitErr("analyzing", err) } @@ -170,7 +170,7 @@ var experiments = []Experiment{ ), } -func analyze(metrics *Metrics, sampler Sampler, shipper indexshipper.IndexShipper, client client.Client, tableName string, tenants []string) error { +func analyze(metrics *Metrics, sampler Sampler, indexShipper indexshipper.IndexShipper, client client.Client, tableName string, tenants []string) error { metrics.tenants.Add(float64(len(tenants))) var n int // count iterated series @@ -178,11 +178,11 @@ func analyze(metrics *Metrics, sampler Sampler, shipper indexshipper.IndexShippe pool := newPool(runtime.NumCPU()) for _, tenant := range tenants { - err := shipper.ForEach( + err := indexShipper.ForEach( context.Background(), tableName, tenant, - indexshipper_index.ForEachIndexCallback(func(isMultiTenantIndex bool, idx indexshipper_index.Index) error { + shipperindex.ForEachIndexCallback(func(isMultiTenantIndex bool, idx shipperindex.Index) error { if isMultiTenantIndex { return nil } @@ -191,14 +191,14 @@ func analyze(metrics *Metrics, sampler Sampler, shipper indexshipper.IndexShippe _ = casted.ForSeries( context.Background(), nil, model.Earliest, model.Latest, - func(ls labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) { - chksCpy := make([]index.ChunkMeta, len(chks)) + func(ls labels.Labels, fp model.Fingerprint, chks []tsdbindex.ChunkMeta) { + chksCpy := make([]tsdbindex.ChunkMeta, len(chks)) copy(chksCpy, chks) pool.acquire( ls.Copy(), fp, chksCpy, - func(ls labels.Labels, fp model.Fingerprint, chks []index.ChunkMeta) { + func(ls labels.Labels, fp model.Fingerprint, chks []tsdbindex.ChunkMeta) { metrics.series.Inc() metrics.chunks.Add(float64(len(chks))) diff --git a/tools/tsdb/helpers/setup.go b/tools/tsdb/helpers/setup.go index 49b2c0cd347d4..95d599e704971 100644 --- a/tools/tsdb/helpers/setup.go +++ b/tools/tsdb/helpers/setup.go @@ -16,7 +16,7 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" "github.com/grafana/loki/pkg/util/cfg" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/pkg/validation" diff --git a/tools/tsdb/index-analyzer/analytics.go b/tools/tsdb/index-analyzer/analytics.go index 4cf998fbee412..7558c17fc8030 100644 --- a/tools/tsdb/index-analyzer/analytics.go +++ b/tools/tsdb/index-analyzer/analytics.go @@ -7,13 +7,13 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/tsdb" - tsdb_index "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + tsdb_index "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) -func analyze(shipper indexshipper.IndexShipper, tableName string, tenants []string) error { +func analyze(indexShipper indexshipper.IndexShipper, tableName string, tenants []string) error { var ( series int @@ -25,7 +25,7 @@ func analyze(shipper indexshipper.IndexShipper, tableName string, tenants []stri ) for _, tenant := range tenants { fmt.Printf("analyzing tenant %s\n", tenant) - err := shipper.ForEach( + err := indexShipper.ForEach( context.Background(), tableName, tenant, diff --git a/tools/tsdb/index-analyzer/main.go b/tools/tsdb/index-analyzer/main.go index 3b42c8cfdc729..316d6211abd80 100644 --- a/tools/tsdb/index-analyzer/main.go +++ b/tools/tsdb/index-analyzer/main.go @@ -7,9 +7,9 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper" - indexshipper_index "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - "github.com/grafana/loki/pkg/storage/stores/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" util_log "github.com/grafana/loki/pkg/util/log" "github.com/grafana/loki/tools/tsdb/helpers" ) @@ -28,7 +28,7 @@ func main() { tableRanges := helpers.GetIndexStoreTableRanges(config.TSDBType, conf.SchemaConfig.Configs) - openFn := func(p string) (indexshipper_index.Index, error) { + openFn := func(p string) (index.Index, error) { return tsdb.OpenShippableTSDB(p, tsdb.IndexOpts{}) } diff --git a/tools/tsdb/migrate-versions/main.go b/tools/tsdb/migrate-versions/main.go index 443e6b766e2af..62682be4884d7 100644 --- a/tools/tsdb/migrate-versions/main.go +++ b/tools/tsdb/migrate-versions/main.go @@ -22,10 +22,10 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk/client/util" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/index" - shipper_storage "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/tsdb" - tsdb_index "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + shipperindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/index" + shipperstorage "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + tsdbindex "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" "github.com/grafana/loki/pkg/util/cfg" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -36,7 +36,7 @@ const ( ) var ( - desiredVer = tsdb_index.FormatV3 + desiredVer = tsdbindex.FormatV3 tableNumMin, tableNumMax int64 newTablePrefix string ) @@ -102,7 +102,7 @@ func migrateTables(pCfg config.PeriodConfig, storageCfg storage.Config, clientMe return err } - indexStorageClient := shipper_storage.NewIndexStorageClient(objClient, storageCfg.TSDBShipperConfig.SharedStoreKeyPrefix) + indexStorageClient := shipperstorage.NewIndexStorageClient(objClient, storageCfg.TSDBShipperConfig.SharedStoreKeyPrefix) tableNames, err := indexStorageClient.ListTables(context.Background()) if err != nil { @@ -142,7 +142,7 @@ func migrateTables(pCfg config.PeriodConfig, storageCfg storage.Config, clientMe return nil } -func migrateTable(tableName string, indexStorageClient shipper_storage.Client) error { +func migrateTable(tableName string, indexStorageClient shipperstorage.Client) error { tempDir := os.TempDir() uncompactedFiles, tenants, err := indexStorageClient.ListFiles(context.Background(), tableName, true) @@ -182,15 +182,15 @@ func migrateTable(tableName string, indexStorageClient shipper_storage.Client) e dst := filepath.Join(tenantDir, indexFiles[0].Name) - decompress := shipper_storage.IsCompressedFile(indexFiles[0].Name) + decompress := shipperstorage.IsCompressedFile(indexFiles[0].Name) if decompress { dst = strings.Trim(dst, gzipExtension) } - if err := shipper_storage.DownloadFileFromStorage( + if err := shipperstorage.DownloadFileFromStorage( dst, decompress, true, - shipper_storage.LoggerWithFilename(util_log.Logger, indexFiles[0].Name), + shipperstorage.LoggerWithFilename(util_log.Logger, indexFiles[0].Name), func() (io.ReadCloser, error) { return indexStorageClient.GetUserFile(context.Background(), tableName, tenant, indexFiles[0].Name) }, @@ -234,7 +234,7 @@ func migrateTable(tableName string, indexStorageClient shipper_storage.Client) e return nil } -func uploadFile(idx index.Index, indexStorageClient shipper_storage.Client, tableName, tenant string) error { +func uploadFile(idx shipperindex.Index, indexStorageClient shipperstorage.Client, tableName, tenant string) error { fileName := idx.Name() level.Debug(util_log.Logger).Log("msg", fmt.Sprintf("uploading index %s", fileName)) diff --git a/tools/tsdb/migrate-versions/main_test.go b/tools/tsdb/migrate-versions/main_test.go index 5fa80cae526b3..9d944695b0f77 100644 --- a/tools/tsdb/migrate-versions/main_test.go +++ b/tools/tsdb/migrate-versions/main_test.go @@ -17,9 +17,9 @@ import ( "github.com/grafana/loki/pkg/storage" "github.com/grafana/loki/pkg/storage/chunk/client/local" "github.com/grafana/loki/pkg/storage/config" - shipper_storage "github.com/grafana/loki/pkg/storage/stores/indexshipper/storage" - "github.com/grafana/loki/pkg/storage/stores/tsdb" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + shipperstorage "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/storage" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" util_log "github.com/grafana/loki/pkg/util/log" ) @@ -52,7 +52,7 @@ func TestMigrateTables(t *testing.T) { objClient, err := storage.NewObjectClient(pcfg.ObjectType, storageCfg, clientMetrics) require.NoError(t, err) - indexStorageClient := shipper_storage.NewIndexStorageClient(objClient, storageCfg.TSDBShipperConfig.SharedStoreKeyPrefix) + indexStorageClient := shipperstorage.NewIndexStorageClient(objClient, storageCfg.TSDBShipperConfig.SharedStoreKeyPrefix) currTableName := pcfg.IndexTables.TableFor(now) currTableNum, err := config.ExtractTableNumberFromName(currTableName) @@ -122,11 +122,11 @@ func TestMigrateTables(t *testing.T) { require.Len(t, indexFiles, 1) dst := filepath.Join(t.TempDir(), strings.Trim(indexFiles[0].Name, gzipExtension)) - err = shipper_storage.DownloadFileFromStorage( + err = shipperstorage.DownloadFileFromStorage( dst, true, true, - shipper_storage.LoggerWithFilename(util_log.Logger, indexFiles[0].Name), + shipperstorage.LoggerWithFilename(util_log.Logger, indexFiles[0].Name), func() (io.ReadCloser, error) { return indexStorageClient.GetUserFile(context.Background(), table, userID, indexFiles[0].Name) }, diff --git a/tools/tsdb/tsdb-map/main.go b/tools/tsdb/tsdb-map/main.go index 9078c877f230e..4c302dbf26101 100644 --- a/tools/tsdb/tsdb-map/main.go +++ b/tools/tsdb/tsdb-map/main.go @@ -12,11 +12,11 @@ import ( "gopkg.in/yaml.v2" "github.com/grafana/loki/pkg/storage/config" - "github.com/grafana/loki/pkg/storage/stores/indexshipper/compactor/retention" - "github.com/grafana/loki/pkg/storage/stores/shipper/index/compactor" - shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util" - "github.com/grafana/loki/pkg/storage/stores/tsdb" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + boltdbcompactor "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb/compactor" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/compactor/retention" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/util" ) var ( @@ -62,7 +62,7 @@ func main() { panic("dest is required") } - db, err := shipper_util.SafeOpenBoltdbFile(*source) + db, err := util.SafeOpenBoltdbFile(*source) if err != nil { panic(err) } @@ -78,7 +78,7 @@ func main() { // loads everything into memory. if err := db.View(func(t *bbolt.Tx) error { - return compactor.ForEachChunk(context.Background(), t.Bucket([]byte("index")), periodConfig, func(entry retention.ChunkEntry) (bool, error) { + return boltdbcompactor.ForEachChunk(context.Background(), t.Bucket([]byte("index")), periodConfig, func(entry retention.ChunkEntry) (bool, error) { builder.AddSeries(entry.Labels, model.Fingerprint(entry.Labels.Hash()), []index.ChunkMeta{{ Checksum: extractChecksumFromChunkID(entry.ChunkID), MinTime: int64(entry.From), diff --git a/tools/tsdb/tsdb-map/main_test.go b/tools/tsdb/tsdb-map/main_test.go index 4d49085290d32..25e358b51b67a 100644 --- a/tools/tsdb/tsdb-map/main_test.go +++ b/tools/tsdb/tsdb-map/main_test.go @@ -11,8 +11,8 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/storage/stores/tsdb" - "github.com/grafana/loki/pkg/storage/stores/tsdb/index" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb" + "github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/tsdb/index" ) func TestExtractChecksum(t *testing.T) {