diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 82f3abed69c7b..04400595b4dbd 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -1779,16 +1779,12 @@ storage: # CLI flag: -common.storage.congestion-control.hedge.strategy [strategy: | default = ""] - object_store: - # The gcs_storage_backend block configures the connection to Google Cloud - # Storage object storage backend. - # The CLI flags prefix for this block configuration is: common.storage - [gcs: ] - - # Prefix for all objects stored in the backend storage. For simplicity, it - # may only contain digits and English alphabet letters. - # CLI flag: -common.storage.object-store.storage-prefix - [storage_prefix: | default = ""] + # The object_store block configures the connection to object storage backends + # using thanos-io/objstore clients. This will become the default way of + # configuring object store clients in future releases. Currently this is + # opt-in and takes effect only when `-use-thanos-objstore` is set to true. + # The CLI flags prefix for this block configuration is: common.storage + [object_store: ] [persist_tokens: ] @@ -4115,6 +4111,22 @@ Named store from this example can be used by setting object_store to store-1 in [cos: ] ``` +### object_store + +The `object_store` block configures the connection to object storage backends using thanos-io/objstore clients. This will become the default way of configuring object store clients in future releases. Currently this is opt-in and takes effect only when `-use-thanos-objstore` is set to true. + +```yaml +# The gcs_storage_backend block configures the connection to Google Cloud +# Storage object storage backend. +# The CLI flags prefix for this block configuration is: common.storage +[gcs: ] + +# Prefix for all objects stored in the backend storage. For simplicity, it may +# only contain digits and English alphabet letters. +# CLI flag: -.object-store.storage-prefix +[storage_prefix: | default = ""] +``` + ### operational_config These are values which allow you to control aspects of Loki's operation, most commonly used for controlling types of higher verbosity logging, the values here can be overridden in the `configs` section of the `runtime_config` file. @@ -5667,19 +5679,18 @@ congestion_control: # CLI flag: -store.max-parallel-get-chunk [max_parallel_get_chunk: | default = 150] -# Enable the thanos.io/objstore to be the backend for object storage +# Enables the use of thanos-io/objstore clients for connecting to object +# storage. When set to true, the configuration inside +# `storage_config.object_store` or `common.storage.object_store` block takes +# effect. # CLI flag: -use-thanos-objstore [use_thanos_objstore: | default = false] -object_store: - # The gcs_storage_backend block configures the connection to Google Cloud - # Storage object storage backend. - [gcs: ] - - # Prefix for all objects stored in the backend storage. For simplicity, it may - # only contain digits and English alphabet letters. - # CLI flag: -object-store.storage-prefix - [storage_prefix: | default = ""] +# The object_store block configures the connection to object storage backends +# using thanos-io/objstore clients. This will become the default way of +# configuring object store clients in future releases. Currently this is opt-in +# and takes effect only when `-use-thanos-objstore` is set to true. +[object_store: ] # The maximum number of chunks to fetch per batch. # CLI flag: -store.max-chunk-batch-size diff --git a/pkg/storage/bucket/client.go b/pkg/storage/bucket/client.go index d81e63be179f5..94433dbd4a769 100644 --- a/pkg/storage/bucket/client.go +++ b/pkg/storage/bucket/client.go @@ -44,7 +44,6 @@ var ( ErrUnsupportedStorageBackend = errors.New("unsupported storage backend") ErrInvalidCharactersInStoragePrefix = errors.New("storage prefix contains invalid characters, it may only contain digits and English alphabet letters") - // TODO: Remove this global if we can tag all clients with component name metrics = objstore.BucketMetrics(prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer), "") ) diff --git a/pkg/storage/chunk/client/object_client.go b/pkg/storage/chunk/client/object_client.go index b2e4f2a0ac4c4..ba1d5be0120be 100644 --- a/pkg/storage/chunk/client/object_client.go +++ b/pkg/storage/chunk/client/object_client.go @@ -183,7 +183,8 @@ func (o *client) getChunk(ctx context.Context, decodeContext *chunk.DecodeContex } defer readCloser.Close() - // reset if GetObject fails to return a valid size + // reset if the size is unknown + // start with a buf of size bytes.MinRead since we cannot avoid allocations if size < 0 { size = 0 } diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index 34b7de4303279..ac6964e4d4dbf 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -325,7 +325,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.Hedging.RegisterFlagsWithPrefix("store.", f) cfg.CongestionControl.RegisterFlagsWithPrefix("store.", f) - f.BoolVar(&cfg.UseThanosObjstore, "use-thanos-objstore", false, "Enable the thanos.io/objstore to be the backend for object storage") + f.BoolVar(&cfg.UseThanosObjstore, "use-thanos-objstore", false, "Enables the use of thanos-io/objstore clients for connecting to object storage. When set to true, the configuration inside `storage_config.object_store` or `common.storage.object_store` block takes effect.") cfg.ObjectStore.RegisterFlags(f) cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "", f) diff --git a/tools/doc-generator/parse/root_blocks.go b/tools/doc-generator/parse/root_blocks.go index e0e22bfceb694..37cc828c26a1f 100644 --- a/tools/doc-generator/parse/root_blocks.go +++ b/tools/doc-generator/parse/root_blocks.go @@ -33,6 +33,7 @@ import ( "github.com/grafana/loki/v3/pkg/runtime" "github.com/grafana/loki/v3/pkg/scheduler" "github.com/grafana/loki/v3/pkg/storage" + "github.com/grafana/loki/v3/pkg/storage/bucket" "github.com/grafana/loki/v3/pkg/storage/bucket/gcs" "github.com/grafana/loki/v3/pkg/storage/chunk/cache" "github.com/grafana/loki/v3/pkg/storage/chunk/client/alibaba" @@ -294,6 +295,11 @@ Named store from this example can be used by setting object_store to store-1 in StructType: []reflect.Type{reflect.TypeOf(push.AttributesConfig{})}, Desc: "Define actions for matching OpenTelemetry (OTEL) attributes.", }, + { + Name: "object_store", + StructType: []reflect.Type{reflect.TypeOf(bucket.Config{})}, + Desc: "The object_store block configures the connection to object storage backends using thanos-io/objstore clients. This will become the default way of configuring object store clients in future releases. Currently this is opt-in and takes effect only when `-use-thanos-objstore` is set to true.", + }, { Name: "gcs_storage_backend", StructType: []reflect.Type{reflect.TypeOf(gcs.Config{})},