From 765638185e13f3d2cf4d5360d2b8aa2a6eea52aa Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 3 Apr 2024 19:09:59 +0000 Subject: [PATCH 01/11] enable structure metadata by default. error if Loki tries to start with structured metadata enabled and a schema less than v13 Signed-off-by: Edward Welch --- docs/sources/configure/_index.md | 4 ++-- pkg/loki/loki.go | 10 ++++++++++ pkg/storage/config/schema_config.go | 2 +- pkg/validation/limits.go | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md index bd561e1116b9a..e0425597ec9bf 100644 --- a/docs/sources/configure/_index.md +++ b/docs/sources/configure/_index.md @@ -3265,7 +3265,7 @@ shard_streams: # Allow user to send structured metadata in push payload. # CLI flag: -validation.allow-structured-metadata -[allow_structured_metadata: | default = false] +[allow_structured_metadata: | default = true] # Maximum size accepted for structured metadata per log line. # CLI flag: -limits.max-structured-metadata-size @@ -4792,7 +4792,7 @@ The `period_config` block configures what index schemas should be used for from # gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc. [object_store: | default = ""] -# The schema version to use, current recommended schema is v12. +# The schema version to use, current recommended schema is v13. [schema: | default = ""] # Configures how the index is updated and stored. diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 536ec5728b053..baadceca273c2 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -273,6 +273,16 @@ func (c *Config) Validate() error { } } + // Schema version 13 is required to use structured metadata + p := config.ActivePeriodConfig(c.SchemaConfig.Configs) + version, err := c.SchemaConfig.Configs[p].VersionAsInt() + if err != nil { + return err + } + if c.LimitsConfig.AllowStructuredMetadata && version < 13 { + return fmt.Errorf("schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema) + } + return nil } diff --git a/pkg/storage/config/schema_config.go b/pkg/storage/config/schema_config.go index c7e72886b738e..9ff620490177b 100644 --- a/pkg/storage/config/schema_config.go +++ b/pkg/storage/config/schema_config.go @@ -161,7 +161,7 @@ type PeriodConfig struct { IndexType string `yaml:"store" doc:"description=store and object_store below affect which key is used. Which index to use. Either tsdb or boltdb-shipper. Following stores are deprecated: aws, aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."` // type of object client to use. ObjectType string `yaml:"object_store" doc:"description=Which store to use for the chunks. Either aws (alias s3), azure, gcs, alibabacloud, bos, cos, swift, filesystem, or a named_store (refer to named_stores_config). Following stores are deprecated: aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."` - Schema string `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v12."` + Schema string `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v13."` IndexTables IndexPeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."` ChunkTables PeriodicTableConfig `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."` RowShards uint32 `yaml:"row_shards" doc:"default=16|description=How many shards will be created. Only used if schema is v10 or greater."` diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index ed8f508447683..a304fcfd362f5 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -382,7 +382,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.VolumeMaxSeries, "limits.volume-max-series", 1000, "The default number of aggregated series or labels that can be returned from a log-volume endpoint") - f.BoolVar(&l.AllowStructuredMetadata, "validation.allow-structured-metadata", false, "Allow user to send structured metadata (non-indexed labels) in push payload.") + f.BoolVar(&l.AllowStructuredMetadata, "validation.allow-structured-metadata", true, "Allow user to send structured metadata (non-indexed labels) in push payload.") _ = l.MaxStructuredMetadataSize.Set(defaultMaxStructuredMetadataSize) f.Var(&l.MaxStructuredMetadataSize, "limits.max-structured-metadata-size", "Maximum size accepted for structured metadata per entry. Default: 64 kb. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") f.IntVar(&l.MaxStructuredMetadataEntriesCount, "limits.max-structured-metadata-entries-count", defaultMaxStructuredMetadataCount, "Maximum number of structured metadata entries per log line. Default: 128. Any log line exceeding this limit will be discarded. There is no limit when unset or set to 0.") From 0c98fa884eabce88ac9b4452c970000fe7d51d4d Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 3 Apr 2024 20:32:10 +0000 Subject: [PATCH 02/11] also require tsdb to use structured metadata Signed-off-by: Edward Welch --- pkg/loki/loki.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index baadceca273c2..3e366f6a3cde3 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -3,6 +3,7 @@ package loki import ( "bytes" "context" + stdlib_errors "errors" "flag" "fmt" "net/http" @@ -273,6 +274,8 @@ func (c *Config) Validate() error { } } + var combinedErrors error + // Schema version 13 is required to use structured metadata p := config.ActivePeriodConfig(c.SchemaConfig.Configs) version, err := c.SchemaConfig.Configs[p].VersionAsInt() @@ -280,10 +283,14 @@ func (c *Config) Validate() error { return err } if c.LimitsConfig.AllowStructuredMetadata && version < 13 { - return fmt.Errorf("schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema) + combinedErrors = stdlib_errors.Join(combinedErrors, fmt.Errorf("CONFIG ERROR: schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema)) + } + // TSDB index is required to use structured metadata + if c.SchemaConfig.Configs[p].IndexType != config.TSDBType { + combinedErrors = stdlib_errors.Join(combinedErrors, fmt.Errorf("CONFIG ERROR: `tsdb` index type is required to store Structured Metadata and use native OTLP ingestion, your index type is `%s` (defined in the `store` parameter of the schema_config). Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update the schema to use index type `tsdb` before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].IndexType)) } - return nil + return combinedErrors } func (c *Config) isModuleEnabled(m string) bool { From d2dbaea3ef4ff5faea88ca5a23765c322a923ccc Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 3 Apr 2024 20:48:08 +0000 Subject: [PATCH 03/11] try to better clarify when there are multiple errors Signed-off-by: Edward Welch --- pkg/loki/loki.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 3e366f6a3cde3..00640d57337c3 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -274,7 +274,7 @@ func (c *Config) Validate() error { } } - var combinedErrors error + var errs []error // Schema version 13 is required to use structured metadata p := config.ActivePeriodConfig(c.SchemaConfig.Configs) @@ -283,14 +283,21 @@ func (c *Config) Validate() error { return err } if c.LimitsConfig.AllowStructuredMetadata && version < 13 { - combinedErrors = stdlib_errors.Join(combinedErrors, fmt.Errorf("CONFIG ERROR: schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema)) + errs = append(errs, fmt.Errorf("CONFIG ERROR: schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema)) } // TSDB index is required to use structured metadata if c.SchemaConfig.Configs[p].IndexType != config.TSDBType { - combinedErrors = stdlib_errors.Join(combinedErrors, fmt.Errorf("CONFIG ERROR: `tsdb` index type is required to store Structured Metadata and use native OTLP ingestion, your index type is `%s` (defined in the `store` parameter of the schema_config). Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update the schema to use index type `tsdb` before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].IndexType)) + errs = append(errs, fmt.Errorf("CONFIG ERROR: `tsdb` index type is required to store Structured Metadata and use native OTLP ingestion, your index type is `%s` (defined in the `store` parameter of the schema_config). Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update the schema to use index type `tsdb` before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].IndexType)) } - return combinedErrors + if len(errs) > 1 { + errs = append([]error{fmt.Errorf("MULTIPLE CONFIG ERRORS FOUND, PLEASE READ CAREFULLY")}, errs...) + return stdlib_errors.Join(errs...) + } else if len(errs) == 1 { + return errs[0] + } + + return nil } func (c *Config) isModuleEnabled(m string) bool { From 67db73092234929bcd2f2bc95a351e59e606b15e Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 00:20:46 +0000 Subject: [PATCH 04/11] upgrade guide Signed-off-by: Edward Welch --- docs/sources/setup/upgrade/_index.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 481cf14182d70..2f76319ed6177 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -174,6 +174,14 @@ This new metric will provide a more clear signal that there is an issue with ing | `legacy-read-mode` | false | true | Deprecated. It will be removed in the next minor release. | {{% /responsive-table %}} +#### Structured Metadata, Open Telemetry, Schemas and Indexes + +A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{ < relref "../../../get-started/labels/structured-metadata.md" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. + +Structured Metadata is enabled by default in Loki 3.0, however, it requires your active schema be using both the `TSDB` index type AND the `v13` storage schema. If you are not using both of these you have two options: + * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../../operations/storage/schema/#changing-the-schema" >}}). + * Disable Structured Metadata (and therefor OTLP support) and upgrade to 3.0 and perform the schema migration after. This can be done by setting `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false`. + #### Automatic stream sharding is enabled by default Automatic stream sharding helps keep the write load of high volume streams balanced across ingesters and helps to avoid hot-spotting. Check out the [operations page](https://grafana.com/docs/loki/latest/operations/automatic-stream-sharding/) for more information From 5d8a8fa0a689a3f2d1a29bdbda4d4f4a1981b249 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 00:48:21 +0000 Subject: [PATCH 05/11] fix links Signed-off-by: Edward Welch --- docs/sources/setup/upgrade/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 2f76319ed6177..56d943ae75eb5 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -176,10 +176,10 @@ This new metric will provide a more clear signal that there is an issue with ing #### Structured Metadata, Open Telemetry, Schemas and Indexes -A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{ < relref "../../../get-started/labels/structured-metadata.md" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. +A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{ < relref "../../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. Structured Metadata is enabled by default in Loki 3.0, however, it requires your active schema be using both the `TSDB` index type AND the `v13` storage schema. If you are not using both of these you have two options: - * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../../operations/storage/schema/#changing-the-schema" >}}). + * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../../operations/storage/schema#changing-the-schema" >}}). * Disable Structured Metadata (and therefor OTLP support) and upgrade to 3.0 and perform the schema migration after. This can be done by setting `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false`. #### Automatic stream sharding is enabled by default From 5a51081c8582492665216ee00b7bfb9f2509e0e8 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 00:55:32 +0000 Subject: [PATCH 06/11] fix links Signed-off-by: Edward Welch --- docs/sources/setup/upgrade/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 56d943ae75eb5..7aa947c00d32f 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -176,10 +176,10 @@ This new metric will provide a more clear signal that there is an issue with ing #### Structured Metadata, Open Telemetry, Schemas and Indexes -A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{ < relref "../../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. +A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{ < relref "../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. Structured Metadata is enabled by default in Loki 3.0, however, it requires your active schema be using both the `TSDB` index type AND the `v13` storage schema. If you are not using both of these you have two options: - * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../../operations/storage/schema#changing-the-schema" >}}). + * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../operations/storage/schema#changing-the-schema" >}}). * Disable Structured Metadata (and therefor OTLP support) and upgrade to 3.0 and perform the schema migration after. This can be done by setting `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false`. #### Automatic stream sharding is enabled by default From 06771f622b9e13fe00558f47f22aeba2b2083844 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 12:31:07 +0000 Subject: [PATCH 07/11] update integration tests Signed-off-by: Edward Welch --- .../loki_micro_services_delete_test.go | 2 +- integration/loki_micro_services_test.go | 30 ++++++++++++++----- integration/loki_rule_eval_test.go | 4 ++- integration/loki_simple_scalable_test.go | 4 ++- integration/loki_single_binary_test.go | 4 ++- integration/per_request_limits_test.go | 4 ++- pkg/loki/loki.go | 2 +- 7 files changed, 37 insertions(+), 13 deletions(-) diff --git a/integration/loki_micro_services_delete_test.go b/integration/loki_micro_services_delete_test.go index ce83cdb4d9f5a..068d10ceb68bb 100644 --- a/integration/loki_micro_services_delete_test.go +++ b/integration/loki_micro_services_delete_test.go @@ -29,7 +29,7 @@ type pushRequest struct { } func TestMicroServicesDeleteRequest(t *testing.T) { - clu := cluster.New(nil, cluster.SchemaWithBoltDBAndBoltDB, func(c *cluster.Cluster) { + clu := cluster.New(nil, cluster.SchemaWithTSDBAndTSDB, func(c *cluster.Cluster) { c.SetSchemaVer("v13") }) defer func() { diff --git a/integration/loki_micro_services_test.go b/integration/loki_micro_services_test.go index 611fafb15ab7a..33f28296bc062 100644 --- a/integration/loki_micro_services_test.go +++ b/integration/loki_micro_services_test.go @@ -30,7 +30,9 @@ import ( ) func TestMicroServicesIngestQuery(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDBAndTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() @@ -227,10 +229,12 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) { "compactor", "-target=compactor", "-compactor.compaction-interval=1s", + "-validation.allow-structured-metadata=false", ) tDistributor = clu.AddComponent( "distributor", "-target=distributor", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -241,11 +245,13 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) { "ingester", "-target=ingester", "-ingester.flush-on-shutdown=true", + "-validation.allow-structured-metadata=false", ) tQueryScheduler = clu.AddComponent( "query-scheduler", "-target=query-scheduler", "-query-scheduler.use-scheduler-ring=false", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -258,12 +264,14 @@ func TestMicroServicesIngestQueryWithSchemaChange(t *testing.T) { "-frontend.scheduler-address="+tQueryScheduler.GRPCURL(), "-frontend.default-validity=0s", "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) tQuerier = clu.AddComponent( "querier", "-target=querier", "-querier.scheduler-address="+tQueryScheduler.GRPCURL(), "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -420,10 +428,12 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) "compactor", "-target=compactor", "-compactor.compaction-interval=1s", + "-validation.allow-structured-metadata=false", ) tDistributor = clu.AddComponent( "distributor", "-target=distributor", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -434,11 +444,13 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) "ingester", "-target=ingester", "-ingester.flush-on-shutdown=true", + "-validation.allow-structured-metadata=false", ) tQueryScheduler = clu.AddComponent( "query-scheduler", "-target=query-scheduler", "-query-scheduler.use-scheduler-ring=false", + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -451,12 +463,14 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) "-frontend.scheduler-address="+tQueryScheduler.GRPCURL(), "-frontend.default-validity=0s", "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) tQuerier = clu.AddComponent( "querier", "-target=querier", "-querier.scheduler-address="+tQueryScheduler.GRPCURL(), "-common.compactor-address="+tCompactor.HTTPURL(), + "-validation.allow-structured-metadata=false", ) ) require.NoError(t, clu.Run()) @@ -472,12 +486,12 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) cliQueryFrontend.Now = now t.Run("ingest-logs", func(t *testing.T) { - require.NoError(t, cliDistributor.PushLogLine("lineA", time.Now().Add(-48*time.Hour), map[string]string{"traceID": "123"}, map[string]string{"job": "fake"})) - require.NoError(t, cliDistributor.PushLogLine("lineB", time.Now().Add(-36*time.Hour), map[string]string{"traceID": "456"}, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineA", time.Now().Add(-48*time.Hour), nil, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineB", time.Now().Add(-36*time.Hour), nil, map[string]string{"job": "fake"})) // ingest logs to the current period - require.NoError(t, cliDistributor.PushLogLine("lineC", now, map[string]string{"traceID": "789"}, map[string]string{"job": "fake"})) - require.NoError(t, cliDistributor.PushLogLine("lineD", now, map[string]string{"traceID": "123"}, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineC", now, nil, map[string]string{"job": "fake"})) + require.NoError(t, cliDistributor.PushLogLine("lineD", now, nil, map[string]string{"job": "fake"})) }) @@ -527,7 +541,9 @@ func TestMicroServicesIngestQueryOverMultipleBucketSingleProvider(t *testing.T) } func TestSchedulerRing(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() @@ -645,7 +661,7 @@ func TestSchedulerRing(t *testing.T) { } func TestOTLPLogsIngestQuery(t *testing.T) { - clu := cluster.New(nil, func(c *cluster.Cluster) { + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { c.SetSchemaVer("v13") }) defer func() { diff --git a/integration/loki_rule_eval_test.go b/integration/loki_rule_eval_test.go index 00caeef8883c2..e0898c50829a3 100644 --- a/integration/loki_rule_eval_test.go +++ b/integration/loki_rule_eval_test.go @@ -36,7 +36,9 @@ func TestRemoteRuleEval(t *testing.T) { // In this test we stub out a remote-write receiver and check that the expected data is sent to it. // Both the local and the remote rule evaluation modes should produce the same result. func testRuleEval(t *testing.T, mode string) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) t.Cleanup(func() { assert.NoError(t, clu.Cleanup()) }) diff --git a/integration/loki_simple_scalable_test.go b/integration/loki_simple_scalable_test.go index 070d3f918a14d..aec97b116fbf6 100644 --- a/integration/loki_simple_scalable_test.go +++ b/integration/loki_simple_scalable_test.go @@ -15,7 +15,9 @@ import ( ) func TestSimpleScalable_IngestQuery(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() diff --git a/integration/loki_single_binary_test.go b/integration/loki_single_binary_test.go index 6aaf64f5b4152..693c946c28646 100644 --- a/integration/loki_single_binary_test.go +++ b/integration/loki_single_binary_test.go @@ -15,7 +15,9 @@ import ( ) func TestSingleBinaryIngestQuery(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() diff --git a/integration/per_request_limits_test.go b/integration/per_request_limits_test.go index 482ff0e93fcfa..935593a04841c 100644 --- a/integration/per_request_limits_test.go +++ b/integration/per_request_limits_test.go @@ -17,7 +17,9 @@ import ( ) func TestPerRequestLimits(t *testing.T) { - clu := cluster.New(nil) + clu := cluster.New(nil, cluster.SchemaWithTSDB, func(c *cluster.Cluster) { + c.SetSchemaVer("v13") + }) defer func() { assert.NoError(t, clu.Cleanup()) }() diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 00640d57337c3..c070cbe1025be 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -286,7 +286,7 @@ func (c *Config) Validate() error { errs = append(errs, fmt.Errorf("CONFIG ERROR: schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is %s. Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].Schema)) } // TSDB index is required to use structured metadata - if c.SchemaConfig.Configs[p].IndexType != config.TSDBType { + if c.LimitsConfig.AllowStructuredMetadata && c.SchemaConfig.Configs[p].IndexType != config.TSDBType { errs = append(errs, fmt.Errorf("CONFIG ERROR: `tsdb` index type is required to store Structured Metadata and use native OTLP ingestion, your index type is `%s` (defined in the `store` parameter of the schema_config). Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update the schema to use index type `tsdb` before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure", c.SchemaConfig.Configs[p].IndexType)) } From f5b1905ef07e436658dc80cbfafe6b75d545988d Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 12:34:47 +0000 Subject: [PATCH 08/11] fix link Signed-off-by: Edward Welch --- docs/sources/setup/upgrade/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/upgrade/_index.md b/docs/sources/setup/upgrade/_index.md index 7aa947c00d32f..133d9493fc53c 100644 --- a/docs/sources/setup/upgrade/_index.md +++ b/docs/sources/setup/upgrade/_index.md @@ -176,7 +176,7 @@ This new metric will provide a more clear signal that there is an issue with ing #### Structured Metadata, Open Telemetry, Schemas and Indexes -A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{ < relref "../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. +A flagship feature of Loki 3.0 is native support for the Open Telemetry Protocol (OTLP). This is made possible by a new feature in Loki called [Structured Metadata]({{< relref "../../get-started/labels/structured-metadata" >}}), a place for metadata which doesn't belong in labels or log lines. OTel resources and attributes are often a great example of data which doesn't belong in the index nor in the log line. Structured Metadata is enabled by default in Loki 3.0, however, it requires your active schema be using both the `TSDB` index type AND the `v13` storage schema. If you are not using both of these you have two options: * Upgrade your index version and schema version before updating to 3.0, see [schema config upgrade]({{< relref "../../operations/storage/schema#changing-the-schema" >}}). From 4c857d04ebf2a016cdee0dd47770b63ce457ddbb Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 13:01:50 +0000 Subject: [PATCH 09/11] remove deprecated config examples Signed-off-by: Edward Welch --- .../examples/configuration-examples.md | 59 ------------------- 1 file changed, 59 deletions(-) diff --git a/docs/sources/configure/examples/configuration-examples.md b/docs/sources/configure/examples/configuration-examples.md index eaaf659049dee..32d6143f84282 100644 --- a/docs/sources/configure/examples/configuration-examples.md +++ b/docs/sources/configure/examples/configuration-examples.md @@ -403,62 +403,3 @@ memberlist: - loki-gossip-ring.loki.svc.cluster.local:7946 # :7946 is the default memberlist port. ``` - - -## 16-(Deprecated)-Cassandra-Snippet.yaml - -```yaml - -# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage -# WARNING - DEPRECATED: The Cassandra index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: cassandra - object_store: filesystem - schema: v12 - index: - prefix: cassandra_table - period: 168h - -storage_config: - cassandra: - username: cassandra - password: cassandra - addresses: 127.0.0.1 - auth: true - keyspace: lokiindex - - filesystem: - directory: /tmp/loki/chunks - - -``` - - -## 17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml - -```yaml - -# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage -# WARNING - DEPRECATED: The DynamoDB index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: aws - object_store: s3 - schema: v12 - index: - prefix: loki_ - -storage_config: - aws: - s3: s3://access_key:secret_access_key@region/bucket_name - dynamodb: - dynamodb_url: dynamodb://access_key:secret_access_key@region - - -``` - From d2f43b068b1cf4f30602a0ba2cc12484ee042236 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 13:08:15 +0000 Subject: [PATCH 10/11] always more steps Signed-off-by: Edward Welch --- .../examples/configuration-examples.md | 1 + .../16-(Deprecated)-Cassandra-Snippet.yaml | 24 ------------------- ...-(Deprecated)-S3-And-DynamoDB-Snippet.yaml | 18 -------------- 3 files changed, 1 insertion(+), 42 deletions(-) delete mode 100644 docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml delete mode 100644 docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml diff --git a/docs/sources/configure/examples/configuration-examples.md b/docs/sources/configure/examples/configuration-examples.md index 32d6143f84282..76b5ffc7a719f 100644 --- a/docs/sources/configure/examples/configuration-examples.md +++ b/docs/sources/configure/examples/configuration-examples.md @@ -403,3 +403,4 @@ memberlist: - loki-gossip-ring.loki.svc.cluster.local:7946 # :7946 is the default memberlist port. ``` + diff --git a/docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml b/docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml deleted file mode 100644 index 71cacb5743687..0000000000000 --- a/docs/sources/configure/examples/yaml/16-(Deprecated)-Cassandra-Snippet.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage -# WARNING - DEPRECATED: The Cassandra index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: cassandra - object_store: filesystem - schema: v12 - index: - prefix: cassandra_table - period: 168h - -storage_config: - cassandra: - username: cassandra - password: cassandra - addresses: 127.0.0.1 - auth: true - keyspace: lokiindex - - filesystem: - directory: /tmp/loki/chunks - \ No newline at end of file diff --git a/docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml b/docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml deleted file mode 100644 index 0b297b02650b6..0000000000000 --- a/docs/sources/configure/examples/yaml/17-(Deprecated)-S3-And-DynamoDB-Snippet.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage -# WARNING - DEPRECATED: The DynamoDB index store is deprecated and will be removed in a future release. - -schema_config: - configs: - - from: 2020-05-15 - store: aws - object_store: s3 - schema: v12 - index: - prefix: loki_ - -storage_config: - aws: - s3: s3://access_key:secret_access_key@region/bucket_name - dynamodb: - dynamodb_url: dynamodb://access_key:secret_access_key@region - \ No newline at end of file From 5f6475bfb58b41a183558b17db7303c91c9cb136 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Thu, 4 Apr 2024 13:35:06 +0000 Subject: [PATCH 11/11] fix test Signed-off-by: Edward Welch --- pkg/loki/config_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/loki/config_test.go b/pkg/loki/config_test.go index 7a29f80bf02b9..0fa36ea2ba9d4 100644 --- a/pkg/loki/config_test.go +++ b/pkg/loki/config_test.go @@ -67,6 +67,8 @@ func TestCrossComponentValidation(t *testing.T) { }, } { tc.base.RegisterFlags(flag.NewFlagSet(tc.desc, 0)) + // This test predates the newer schema required for structured metadata + tc.base.LimitsConfig.AllowStructuredMetadata = false err := tc.base.Validate() if tc.err { require.NotNil(t, err)