Skip to content

Commit

Permalink
update docs and comments
Browse files Browse the repository at this point in the history
  • Loading branch information
ashwanthgoli committed Oct 18, 2024
1 parent 6050e0b commit 45d33c3
Show file tree
Hide file tree
Showing 5 changed files with 40 additions and 23 deletions.
51 changes: 31 additions & 20 deletions docs/sources/shared/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1779,16 +1779,12 @@ storage:
# CLI flag: -common.storage.congestion-control.hedge.strategy
[strategy: <string> | default = ""]
object_store:
# The gcs_storage_backend block configures the connection to Google Cloud
# Storage object storage backend.
# The CLI flags prefix for this block configuration is: common.storage
[gcs: <gcs_storage_backend>]
# Prefix for all objects stored in the backend storage. For simplicity, it
# may only contain digits and English alphabet letters.
# CLI flag: -common.storage.object-store.storage-prefix
[storage_prefix: <string> | default = ""]
# The object_store block configures the connection to object storage backends
# using thanos-io/objstore clients. This will become the default way of
# configuring object store clients in future releases. Currently this is
# opt-in and takes effect only when `-use-thanos-objstore` is set to true.
# The CLI flags prefix for this block configuration is: common.storage
[object_store: <object_store>]

[persist_tokens: <boolean>]

Expand Down Expand Up @@ -4115,6 +4111,22 @@ Named store from this example can be used by setting object_store to store-1 in
[cos: <map of string to cos_storage_config>]
```

### object_store

The `object_store` block configures the connection to object storage backends using thanos-io/objstore clients. This will become the default way of configuring object store clients in future releases. Currently this is opt-in and takes effect only when `-use-thanos-objstore` is set to true.

```yaml
# The gcs_storage_backend block configures the connection to Google Cloud
# Storage object storage backend.
# The CLI flags prefix for this block configuration is: common.storage
[gcs: <gcs_storage_backend>]
# Prefix for all objects stored in the backend storage. For simplicity, it may
# only contain digits and English alphabet letters.
# CLI flag: -<prefix>.object-store.storage-prefix
[storage_prefix: <string> | default = ""]
```

### operational_config

These are values which allow you to control aspects of Loki's operation, most commonly used for controlling types of higher verbosity logging, the values here can be overridden in the `configs` section of the `runtime_config` file.
Expand Down Expand Up @@ -5667,19 +5679,18 @@ congestion_control:
# CLI flag: -store.max-parallel-get-chunk
[max_parallel_get_chunk: <int> | default = 150]

# Enable the thanos.io/objstore to be the backend for object storage
# Enables the use of thanos-io/objstore clients for connecting to object
# storage. When set to true, the configuration inside
# `storage_config.object_store` or `common.storage.object_store` block takes
# effect.
# CLI flag: -use-thanos-objstore
[use_thanos_objstore: <boolean> | default = false]

object_store:
# The gcs_storage_backend block configures the connection to Google Cloud
# Storage object storage backend.
[gcs: <gcs_storage_backend>]

# Prefix for all objects stored in the backend storage. For simplicity, it may
# only contain digits and English alphabet letters.
# CLI flag: -object-store.storage-prefix
[storage_prefix: <string> | default = ""]
# The object_store block configures the connection to object storage backends
# using thanos-io/objstore clients. This will become the default way of
# configuring object store clients in future releases. Currently this is opt-in
# and takes effect only when `-use-thanos-objstore` is set to true.
[object_store: <object_store>]

# The maximum number of chunks to fetch per batch.
# CLI flag: -store.max-chunk-batch-size
Expand Down
1 change: 0 additions & 1 deletion pkg/storage/bucket/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ var (
ErrUnsupportedStorageBackend = errors.New("unsupported storage backend")
ErrInvalidCharactersInStoragePrefix = errors.New("storage prefix contains invalid characters, it may only contain digits and English alphabet letters")

// TODO: Remove this global if we can tag all clients with component name
metrics = objstore.BucketMetrics(prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer), "")
)

Expand Down
3 changes: 2 additions & 1 deletion pkg/storage/chunk/client/object_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,8 @@ func (o *client) getChunk(ctx context.Context, decodeContext *chunk.DecodeContex
}
defer readCloser.Close()

// reset if GetObject fails to return a valid size
// reset if the size is unknown
// start with a buf of size bytes.MinRead since we cannot avoid allocations
if size < 0 {
size = 0
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/storage/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
cfg.Hedging.RegisterFlagsWithPrefix("store.", f)
cfg.CongestionControl.RegisterFlagsWithPrefix("store.", f)

f.BoolVar(&cfg.UseThanosObjstore, "use-thanos-objstore", false, "Enable the thanos.io/objstore to be the backend for object storage")
f.BoolVar(&cfg.UseThanosObjstore, "use-thanos-objstore", false, "Enables the use of thanos-io/objstore clients for connecting to object storage. When set to true, the configuration inside `storage_config.object_store` or `common.storage.object_store` block takes effect.")
cfg.ObjectStore.RegisterFlags(f)

cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "", f)
Expand Down
6 changes: 6 additions & 0 deletions tools/doc-generator/parse/root_blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/grafana/loki/v3/pkg/runtime"
"github.com/grafana/loki/v3/pkg/scheduler"
"github.com/grafana/loki/v3/pkg/storage"
"github.com/grafana/loki/v3/pkg/storage/bucket"
"github.com/grafana/loki/v3/pkg/storage/bucket/gcs"
"github.com/grafana/loki/v3/pkg/storage/chunk/cache"
"github.com/grafana/loki/v3/pkg/storage/chunk/client/alibaba"
Expand Down Expand Up @@ -294,6 +295,11 @@ Named store from this example can be used by setting object_store to store-1 in
StructType: []reflect.Type{reflect.TypeOf(push.AttributesConfig{})},
Desc: "Define actions for matching OpenTelemetry (OTEL) attributes.",
},
{
Name: "object_store",
StructType: []reflect.Type{reflect.TypeOf(bucket.Config{})},
Desc: "The object_store block configures the connection to object storage backends using thanos-io/objstore clients. This will become the default way of configuring object store clients in future releases. Currently this is opt-in and takes effect only when `-use-thanos-objstore` is set to true.",
},
{
Name: "gcs_storage_backend",
StructType: []reflect.Type{reflect.TypeOf(gcs.Config{})},
Expand Down

0 comments on commit 45d33c3

Please sign in to comment.