chore: add more validation at startup to make sure we have a proper s…

…chema, and necessary directories are created (#12496) Signed-off-by: Edward Welch <[email protected]> Co-authored-by: Dylan Guedes <[email protected]>
grafana · Apr 7, 2024 · 6700413 · 6700413
1 parent 712a2b6
commit 6700413
Show file tree

Hide file tree

Showing 41 changed files with 459 additions and 319 deletions.
diff --git a/docs/sources/configure/examples/configuration-examples.md b/docs/sources/configure/examples/configuration-examples.md
@@ -36,9 +36,6 @@ schema_config:
       period: 24h
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /tmp/loki/index
-    cache_location: /tmp/loki/index_cache
   filesystem:
     directory: /tmp/loki/chunks
 
@@ -148,6 +145,8 @@ storage_config:
 
 # This is a partial configuration to deploy Loki backed by Baidu Object Storage (BOS).
 # The index will be shipped to the storage via tsdb-shipper.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -160,9 +159,6 @@ schema_config:
         period: 24h
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com
@@ -190,6 +186,9 @@ compactor:
 
 ```yaml
 
+common:
+  path_prefix: /tmp/loki
+
 schema_config:
   configs:
     # Starting from 2018-04-15 Loki should store indexes on BoltDB with the v11 schema
@@ -221,6 +220,8 @@ schema_config:
 ```yaml
 
 # This partial configuration uses Alibaba for chunk storage.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -291,6 +292,8 @@ storage_config:
 ```yaml
 
 # This partial configuration uses IBM Cloud Object Storage (COS) for chunk storage. HMAC will be used for authenticating with COS.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -303,9 +306,6 @@ schema_config:
         prefix: index_
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -321,6 +321,8 @@ storage_config:
 ```yaml
 
 # This partial configuration uses IBM Cloud Object Storage (COS) for chunk storage. APIKey will be used for authenticating with COS.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -333,9 +335,6 @@ schema_config:
         prefix: index_
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -358,6 +357,8 @@ storage_config:
 # the same trusted profile.
 # In order to use trusted profile authentication we need to follow an additional step to create a trusted profile.
 # For more details about creating a trusted profile, see https://cloud.ibm.com/docs/account?topic=account-create-trusted-profile&interface=ui.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -370,9 +371,6 @@ schema_config:
         prefix: index_
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/yaml/1-Local-Configuration-Example.yaml b/docs/sources/configure/examples/yaml/1-Local-Configuration-Example.yaml
@@ -25,8 +25,5 @@ schema_config:
       period: 24h
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /tmp/loki/index
-    cache_location: /tmp/loki/index_cache
   filesystem:
     directory: /tmp/loki/chunks
diff --git a/docs/sources/configure/examples/yaml/11-COS-HMAC-Example.yaml b/docs/sources/configure/examples/yaml/11-COS-HMAC-Example.yaml
@@ -1,4 +1,6 @@
 # This partial configuration uses IBM Cloud Object Storage (COS) for chunk storage. HMAC will be used for authenticating with COS.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -11,9 +13,6 @@ schema_config:
         prefix: index_
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/yaml/12-COS-APIKey-Example.yaml b/docs/sources/configure/examples/yaml/12-COS-APIKey-Example.yaml
@@ -1,4 +1,6 @@
 # This partial configuration uses IBM Cloud Object Storage (COS) for chunk storage. APIKey will be used for authenticating with COS.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -11,9 +13,6 @@ schema_config:
         prefix: index_
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/yaml/13-COS-Trusted-Profile-Example.yaml b/docs/sources/configure/examples/yaml/13-COS-Trusted-Profile-Example.yaml
@@ -5,6 +5,8 @@
 # the same trusted profile.
 # In order to use trusted profile authentication we need to follow an additional step to create a trusted profile.
 # For more details about creating a trusted profile, see https://cloud.ibm.com/docs/account?topic=account-create-trusted-profile&interface=ui.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -17,9 +19,6 @@ schema_config:
         prefix: index_
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/yaml/5-BOS-Example.yaml b/docs/sources/configure/examples/yaml/5-BOS-Example.yaml
@@ -1,5 +1,7 @@
 # This is a partial configuration to deploy Loki backed by Baidu Object Storage (BOS).
 # The index will be shipped to the storage via tsdb-shipper.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
@@ -12,9 +14,6 @@ schema_config:
         period: 24h
 
 storage_config:
-  tsdb_shipper:
-    active_index_directory: /loki/index
-    cache_location: /loki/index_cache
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com

diff --git a/docs/sources/configure/examples/yaml/7-Schema-Migration-Snippet.yaml b/docs/sources/configure/examples/yaml/7-Schema-Migration-Snippet.yaml
@@ -1,3 +1,6 @@
+common:
+  path_prefix: /tmp/loki
+
 schema_config:
   configs:
     # Starting from 2018-04-15 Loki should store indexes on BoltDB with the v11 schema

diff --git a/docs/sources/configure/examples/yaml/8-alibaba-cloud-storage-Snippet.yaml b/docs/sources/configure/examples/yaml/8-alibaba-cloud-storage-Snippet.yaml
@@ -1,4 +1,6 @@
 # This partial configuration uses Alibaba for chunk storage.
+common:
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:

diff --git a/pkg/bloomcompactor/retention_test.go b/pkg/bloomcompactor/retention_test.go
@@ -22,6 +22,7 @@ import (
 	storageconfig "github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper/config"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 	util_log "github.com/grafana/loki/v3/pkg/util/log"
 	lokiring "github.com/grafana/loki/v3/pkg/util/ring"
 	"github.com/grafana/loki/v3/pkg/validation"
@@ -773,7 +774,7 @@ func NewMockBloomStoreWithWorkDir(t *testing.T, workDir string) (*bloomshipper.B
 	schemaCfg := storageconfig.SchemaConfig{
 		Configs: []storageconfig.PeriodConfig{
 			{
-				ObjectType: storageconfig.StorageTypeFileSystem,
+				ObjectType: types.StorageTypeFileSystem,
 				From: storageconfig.DayTime{
 					Time: testTime.Add(-2 * 365 * 24 * time.Hour), // -2 year
 				},
@@ -784,7 +785,7 @@ func NewMockBloomStoreWithWorkDir(t *testing.T, workDir string) (*bloomshipper.B
 					}},
 			},
 			{
-				ObjectType: storageconfig.StorageTypeFileSystem,
+				ObjectType: types.StorageTypeFileSystem,
 				From: storageconfig.DayTime{
 					Time: testTime.Add(-365 * 24 * time.Hour), // -1 year
 				},

diff --git a/pkg/bloomcompactor/tsdb.go b/pkg/bloomcompactor/tsdb.go
@@ -22,6 +22,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 )
 
 const (
@@ -185,7 +186,7 @@ func NewTSDBStores(
 	}
 
 	for i, cfg := range schemaCfg.Configs {
-		if cfg.IndexType == config.TSDBType {
+		if cfg.IndexType == types.TSDBType {
 
 			c, err := baseStore.NewObjectClient(cfg.ObjectType, storeCfg, clientMetrics)
 			if err != nil {

diff --git a/pkg/bloomgateway/bloomgateway_test.go b/pkg/bloomgateway/bloomgateway_test.go
@@ -26,6 +26,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper"
 	bloomshipperconfig "github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper/config"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 	"github.com/grafana/loki/v3/pkg/validation"
 )
 
@@ -58,8 +59,8 @@ func setupBloomStore(t *testing.T) *bloomshipper.BloomStore {
 				Period: 24 * time.Hour,
 			},
 		},
-		IndexType:  config.TSDBType,
-		ObjectType: config.StorageTypeFileSystem,
+		IndexType:  types.TSDBType,
+		ObjectType: types.StorageTypeFileSystem,
 		Schema:     "v13",
 		RowShards:  16,
 	}

diff --git a/pkg/ingester/index/multi.go b/pkg/ingester/index/multi.go
@@ -10,6 +10,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/logproto"
 	"github.com/grafana/loki/v3/pkg/logql"
 	"github.com/grafana/loki/v3/pkg/storage/config"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 )
 
 type periodIndex struct {
@@ -34,7 +35,7 @@ func NewMultiInvertedIndex(periods []config.PeriodConfig, indexShards uint32) (*
 
 	for _, pd := range periods {
 		switch pd.IndexType {
-		case config.TSDBType:
+		case types.TSDBType:
 			if bitPrefixed == nil {
 				bitPrefixed, err = NewBitPrefixWithShards(indexShards)
 				if err != nil {

diff --git a/pkg/ingester/index/multi_test.go b/pkg/ingester/index/multi_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/logql"
 	"github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/index"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 )
 
 func MustParseDayTime(s string) config.DayTime {
@@ -26,19 +27,19 @@ func MustParseDayTime(s string) config.DayTime {
 var testPeriodConfigs = []config.PeriodConfig{
 	{
 		From:      MustParseDayTime("2020-01-01"),
-		IndexType: config.StorageTypeBigTable,
+		IndexType: types.StorageTypeBigTable,
 	},
 	{
 		From:      MustParseDayTime("2021-01-01"),
-		IndexType: config.TSDBType,
+		IndexType: types.TSDBType,
 	},
 	{
 		From:      MustParseDayTime("2022-01-01"),
-		IndexType: config.BoltDBShipperType,
+		IndexType: types.BoltDBShipperType,
 	},
 	{
 		From:      MustParseDayTime("2023-01-01"),
-		IndexType: config.TSDBType,
+		IndexType: types.TSDBType,
 	},
 }
 
@@ -50,7 +51,7 @@ func TestIgnoresInvalidShardFactorWhenTSDBNotPresent(t *testing.T) {
 		[]config.PeriodConfig{
 			{
 				From:      MustParseDayTime("2020-01-01"),
-				IndexType: config.StorageTypeBigTable,
+				IndexType: types.StorageTypeBigTable,
 			},
 		},
 		factor,
@@ -61,11 +62,11 @@ func TestIgnoresInvalidShardFactorWhenTSDBNotPresent(t *testing.T) {
 		[]config.PeriodConfig{
 			{
 				From:      MustParseDayTime("2020-01-01"),
-				IndexType: config.StorageTypeBigTable,
+				IndexType: types.StorageTypeBigTable,
 			},
 			{
 				From:      MustParseDayTime("2021-01-01"),
-				IndexType: config.TSDBType,
+				IndexType: types.TSDBType,
 			},
 		},
 		factor,

diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/grafana/loki/v3/pkg/logqlmodel/metadata"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 
 	lokilog "github.com/grafana/loki/v3/pkg/logql/log"
 
@@ -998,13 +999,13 @@ func (i *Ingester) QuerySample(req *logproto.SampleQueryRequest, queryServer log
 func (i *Ingester) asyncStoreMaxLookBack() time.Duration {
 	activePeriodicConfigIndex := config.ActivePeriodConfig(i.periodicConfigs)
 	activePeriodicConfig := i.periodicConfigs[activePeriodicConfigIndex]
-	if activePeriodicConfig.IndexType != config.BoltDBShipperType && activePeriodicConfig.IndexType != config.TSDBType {
+	if activePeriodicConfig.IndexType != types.BoltDBShipperType && activePeriodicConfig.IndexType != types.TSDBType {
 		return 0
 	}
 
 	startTime := activePeriodicConfig.From
-	if activePeriodicConfigIndex != 0 && (i.periodicConfigs[activePeriodicConfigIndex-1].IndexType == config.BoltDBShipperType ||
-		i.periodicConfigs[activePeriodicConfigIndex-1].IndexType == config.TSDBType) {
+	if activePeriodicConfigIndex != 0 && (i.periodicConfigs[activePeriodicConfigIndex-1].IndexType == types.BoltDBShipperType ||
+		i.periodicConfigs[activePeriodicConfigIndex-1].IndexType == types.TSDBType) {
 		startTime = i.periodicConfigs[activePeriodicConfigIndex-1].From
 	}
 

diff --git a/pkg/ingester/instance_test.go b/pkg/ingester/instance_test.go
@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/grafana/loki/v3/pkg/storage/types"
 	"github.com/grafana/loki/v3/pkg/util/httpreq"
 
 	"github.com/grafana/dskit/tenant"
@@ -60,7 +61,7 @@ func MustParseDayTime(s string) config.DayTime {
 var defaultPeriodConfigs = []config.PeriodConfig{
 	{
 		From:      MustParseDayTime("1900-01-01"),
-		IndexType: config.StorageTypeBigTable,
+		IndexType: types.StorageTypeBigTable,
 		Schema:    "v13",
 	},
 }

diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go
@@ -26,7 +26,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/storage"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/client"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/client/local"
-	"github.com/grafana/loki/v3/pkg/storage/config"
+	"github.com/grafana/loki/v3/pkg/storage/types"
 	"github.com/grafana/loki/v3/pkg/util/marshal"
 )
 
@@ -550,7 +550,7 @@ func setupTestEnv(t *testing.T) (string, client.ObjectClient) {
 		},
 	}
 
-	client, err := GetObjectClient(config.StorageTypeFileSystem, conf, cm)
+	client, err := GetObjectClient(types.StorageTypeFileSystem, conf, cm)
 	require.NoError(t, err)
 	require.NotNil(t, client)