Revert "Remove shared_store and shared_store_key_prefix from shipper …

…and compactor (#10840)"

This reverts commit 037dbda.

.github/pull_request_template.md

@@ -13,4 +13,4 @@ Fixes #<issue number>
   - [ ] If the change is worth mentioning in the release notes, add `add-to-release-notes` label
 - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md`
 - [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213)
-- [ ] If the change is deprecating or removing a configuration option, update the `deprecated-config.yaml` and `deleted-config.yaml` files respectively in the `tools/deprecated-config-checker` directory. [Example PR](https://github.com/grafana/loki/pull/10840/commits/0d4416a4b03739583349934b96f272fb4f685d15)
+- [ ] If the change is deprecating or removing a configuration option, update the `deprecated-config.yaml` and `deleted-config.yaml` files respectively in the `tools/deprecated-config-checker` directory. <!-- TODO(salvacorts): Add example PR -->

CHANGELOG.md

@@ -25,7 +25,6 @@
 * [10709](https://github.com/grafana/loki/pull/10709) **chaudum**/**salvacorts** Remove `ingester.max-transfer-retries` configuration option in favor of using the WAL.
 * [10736](https://github.com/grafana/loki/pull/10736) **ashwanthgoli** Deprecate write dedupe cache as this is not required by the newer single store indexes (tsdb and boltdb-shipper).
 * [10693](https://github.com/grafana/loki/pull/10693) **ashwanthgoli** Embedded cache: Updates the metric prefix from `querier_cache_` to `loki_embeddedcache_` and removes duplicate metrics.
-* [10840](https://github.com/grafana/loki/pull/10840) **ashwanthgoli** Removes `shared_store` and `shared_store_key_prefix` from tsdb, boltdb shipper and compactor configs and their corresponding CLI flags.
 * [10793](https://github.com/grafana/loki/pull/10793) **ashwanthgoli** Config: Better configuration defaults to provide a better experience for users out of the box.
 * [10785](https://github.com/grafana/loki/pull/10785) **ashwanthgoli** Config: Removes `querier.worker-parallelism` and updates default value of `querier.max-concurrent` to 4.
 * [10733](https://github.com/grafana/loki/pull/10733) **shantanualsi** Add support for case-insensitive logql funtions

cmd/logcli/main.go

@@ -560,8 +560,7 @@ func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
 	cmd.Flag("include-label", "Include labels given the provided key during output.").StringsVar(&q.ShowLabelsKey)
 	cmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").IntVar(&q.FixedLabelsLen)
 	cmd.Flag("store-config", "Execute the current query using a configured storage from a given Loki configuration file.").Default("").StringVar(&q.LocalConfig)
-	cmd.Flag("remote-schema", "Execute the current query using a remote schema retrieved from the configured -schema-store.").Default("false").BoolVar(&q.FetchSchemaFromStorage)
-	cmd.Flag("schema-store", "Store used for retrieving remote schema.").Default("").StringVar(&q.SchemaStore)
+	cmd.Flag("remote-schema", "Execute the current query using a remote schema retrieved using the configured storage in the given Loki configuration file.").Default("false").BoolVar(&q.FetchSchemaFromStorage)
 	cmd.Flag("colored-output", "Show output with colored labels").Default("false").BoolVar(&q.ColoredOutput)
 
 	return q

docs/sources/configure/_index.md

@@ -2122,6 +2122,17 @@ boltdb_shipper:
   # CLI flag: -boltdb.shipper.active-index-directory
   [active_index_directory: <string> | default = ""]
 
+  # Shared store for keeping index files. Supported types: gcs, s3, azure, cos,
+  # filesystem
+  # CLI flag: -boltdb.shipper.shared-store
+  [shared_store: <string> | default = ""]
+
+  # Prefix to add to Object Keys in Shared store. Path separator(if any) should
+  # always be a '/'. Prefix should never start with a separator but should
+  # always end with it
+  # CLI flag: -boltdb.shipper.shared-store.key-prefix
+  [shared_store_key_prefix: <string> | default = "index/"]
+
   # Cache location for restoring index files from storage for queries
   # CLI flag: -boltdb.shipper.cache-location
   [cache_location: <string> | default = ""]
@@ -2177,6 +2188,17 @@ tsdb_shipper:
   # CLI flag: -tsdb.shipper.active-index-directory
   [active_index_directory: <string> | default = ""]
 
+  # Shared store for keeping index files. Supported types: gcs, s3, azure, cos,
+  # filesystem
+  # CLI flag: -tsdb.shipper.shared-store
+  [shared_store: <string> | default = ""]
+
+  # Prefix to add to Object Keys in Shared store. Path separator(if any) should
+  # always be a '/'. Prefix should never start with a separator but should
+  # always end with it
+  # CLI flag: -tsdb.shipper.shared-store.key-prefix
+  [shared_store_key_prefix: <string> | default = "index/"]
+
   # Cache location for restoring index files from storage for queries
   # CLI flag: -tsdb.shipper.cache-location
   [cache_location: <string> | default = ""]
@@ -2269,6 +2291,19 @@ The `compactor` block configures the compactor component, which compacts index s
 # CLI flag: -compactor.working-directory
 [working_directory: <string> | default = ""]
 
+# The shared store used for storing boltdb files. Supported types: gcs, s3,
+# azure, swift, filesystem, bos, cos. If not set, compactor will be initialized
+# to operate on all the object stores that contain either boltdb-shipper or tsdb
+# index.
+# CLI flag: -compactor.shared-store
+[shared_store: <string> | default = ""]
+
+# Prefix to add to object keys in shared store. Path separator(if any) should
+# always be a '/'. Prefix should never start with a separator but should always
+# end with it.
+# CLI flag: -compactor.shared-store.key-prefix
+[shared_store_key_prefix: <string> | default = "index/"]
+
 # Interval at which to re-run the compaction operation.
 # CLI flag: -compactor.compaction-interval
 [compaction_interval: <duration> | default = 10m]
@@ -2296,14 +2331,10 @@ The `compactor` block configures the compactor component, which compacts index s
 # CLI flag: -compactor.retention-table-timeout
 [retention_table_timeout: <duration> | default = 0s]
 
-# Store used for managing delete requests.
+# Store used for managing delete requests. Defaults to -compactor.shared-store.
 # CLI flag: -compactor.delete-request-store
 [delete_request_store: <string> | default = ""]
 
-# Path prefix for storing delete requests.
-# CLI flag: -compactor.delete-request-store.key-prefix
-[delete_request_store_key_prefix: <string> | default = "index/"]
-
 # The max number of delete requests to run per compaction cycle.
 # CLI flag: -compactor.delete-batch-size
 [delete_batch_size: <int> | default = 70]
@@ -3547,7 +3578,7 @@ ring:
   # CLI flag: -common.storage.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-[instance_interface_names: <list of strings> | default = [<private network interfaces>]]
+[instance_interface_names: <list of strings>]
 
 [instance_addr: <string> | default = ""]
 
@@ -4333,10 +4364,6 @@ The `period_config` block configures what index schemas should be used for from
 
 # Configures how the index is updated and stored.
 index:
-  # Path prefix for index tables. Prefix always needs to end with a path
-  # delimiter '/', except when the prefix is empty.
-  [path_prefix: <string> | default = "index/"]
-
   # Table prefix for all period tables.
   [prefix: <string> | default = ""]
 

docs/sources/configure/examples/configuration-examples.md

@@ -39,6 +39,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /tmp/loki/index
     cache_location: /tmp/loki/index_cache
+    shared_store: filesystem
   filesystem:
     directory: /tmp/loki/chunks
 
@@ -80,6 +81,7 @@ storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
+   shared_store: s3
  aws:
    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
    s3forcepathstyle: true
@@ -136,8 +138,10 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: gcs
   gcs:
     bucket_name: replace_by_your_bucked_name
+
 
 ```
 
@@ -163,6 +167,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: bos
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com
@@ -181,6 +186,7 @@ storage_config:
 
 compactor:
   working_directory: /tmp/loki/compactor
+  shared_store: s3
   compaction_interval: 5m
 
 ```
@@ -212,6 +218,7 @@ schema_config:
     index:
         period: 24h
         prefix: index_
+
 
 ```
 
@@ -236,6 +243,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: alibabacloud
   alibabacloud:
     bucket: <bucket>
     endpoint: <endpoint>
@@ -306,6 +314,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -336,6 +345,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -373,6 +383,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/yaml/1-Local-Configuration-Example.yaml

@@ -28,5 +28,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /tmp/loki/index
     cache_location: /tmp/loki/index_cache
+    shared_store: filesystem
   filesystem:
-    directory: /tmp/loki/chunks
+    directory: /tmp/loki/chunks

docs/sources/configure/examples/yaml/11-COS-HMAC-Example.yaml

@@ -14,6 +14,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/yaml/12-COS-APIKey-Example.yaml

@@ -14,6 +14,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/yaml/13-COS-Trusted-Profile-Example.yaml

@@ -20,6 +20,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/yaml/2-S3-Cluster-Example.yaml

@@ -29,6 +29,7 @@ storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
+   shared_store: s3
  aws:
    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
-   s3forcepathstyle: true
+   s3forcepathstyle: true

docs/sources/configure/examples/yaml/4-GCS-Example.yaml

@@ -28,5 +28,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: gcs
   gcs:
     bucket_name: replace_by_your_bucked_name
+

docs/sources/configure/examples/yaml/5-BOS-Example.yaml

@@ -15,6 +15,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: bos
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com

docs/sources/configure/examples/yaml/6-Compactor-Snippet.yaml

@@ -4,4 +4,5 @@
 
 compactor:
   working_directory: /tmp/loki/compactor
-  compaction_interval: 5m
+  shared_store: s3
+  compaction_interval: 5m

docs/sources/configure/examples/yaml/7-Schema-Migration-Snippet.yaml

@@ -20,3 +20,4 @@ schema_config:
     index:
         period: 24h
         prefix: index_
+

docs/sources/configure/examples/yaml/8-alibaba-cloud-storage-Snippet.yaml

@@ -14,6 +14,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: alibabacloud
   alibabacloud:
     bucket: <bucket>
     endpoint: <endpoint>

docs/sources/operations/storage/boltdb-shipper.md

@@ -41,6 +41,7 @@ storage_config:
 
   boltdb_shipper:
     active_index_directory: /loki/index
+    shared_store: gcs
     cache_location: /loki/boltdb-cache
 ```
 
@@ -143,6 +144,7 @@ The compactor is an optional but suggested component that combines and deduplica
 ```yaml
 compactor:
   working_directory: /loki/compactor
+  shared_store: gcs
 
 storage_config:
   gcs:

docs/sources/operations/storage/logs-deletion.md

@@ -9,7 +9,7 @@ weight: 700
 Grafana Loki supports the deletion of log entries from a specified stream.
 Log entries that fall within a specified time window and match an optional line filter are those that will be deleted.
 
-Log entry deletion is supported _only_ when TSDB or BoltDB shipper is configured as the index store.
+Log entry deletion is supported _only_ when the BoltDB Shipper is configured for the index store.
 
 The compactor component exposes REST [endpoints]({{< relref "../../reference/api#compactor" >}}) that process delete requests.
 Hitting the endpoint specifies the streams and the time window.
@@ -20,7 +20,6 @@ Log entry deletion relies on configuration of the custom logs retention workflow
 ## Configuration
 
 Enable log entry deletion by setting `retention_enabled` to true in the compactor's configuration and setting and `deletion_mode` to `filter-only` or `filter-and-delete` in the runtime config.
-`delete_request_store` also needs to be configured when retention is enabled to process delete requests, this determines the storage bucket that stores the delete requests.
 
 > **Warning:** Be very careful when enabling retention. It is strongly recommended that you also enable versioning on your objects in object storage to allow you to recover from accidental misconfiguration of a retention setting. If you want to enable deletion but not not want to enforce retention, configure the `retention_period` setting with a value of `0s`.
 

docs/sources/operations/storage/retention.md

@@ -60,11 +60,11 @@ This Compactor configuration example activates retention.
 ```yaml
 compactor:
   working_directory: /data/retention
+  shared_store: gcs
   compaction_interval: 10m
   retention_enabled: true
   retention_delete_delay: 2h
   retention_delete_worker_count: 150
-  delete_request_store: gcs
 schema_config:
     configs:
       - from: "2020-07-31"
@@ -78,6 +78,7 @@ storage_config:
     tsdb_shipper:
         active_index_directory: /data/index
         cache_location: /data/index_cache
+        shared_store: gcs
     gcs:
         bucket_name: loki
 ```
@@ -88,8 +89,6 @@ Retention is only available if the index period is 24h. Single store TSDB and si
 
 `retention_enabled` should be set to true. Without this, the Compactor will only compact tables.
 
-`delete_request_store` should be set to configure the store for delete requests. This is required when retention is enabled.
-
 `working_directory` is the directory where marked chunks and temporary tables will be saved.
 
 `compaction_interval` dictates how often compaction and/or retention is applied. If the Compactor falls behind, compaction and/or retention occur as soon as possible.
@@ -247,6 +246,7 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
+    shared_store: gcs
   gcs:
     bucket_name: GCS_BUCKET_NAME
 
@@ -256,6 +256,6 @@ limits_config:
 
 compactor:
   working_directory: /data/retention
-  delete_request_store: gcs
+  shared_store: gcs
   retention_enabled: true
 ```

docs/sources/operations/storage/tsdb.md

@@ -42,6 +42,7 @@ storage_config:
       # only applicable if using microservices where index-gateways are independently deployed.
       # This example is using kubernetes-style naming.
       server_address: dns:///index-gateway.<namespace>.svc.cluster.local:9095
+    shared_store: gcs
   # New tsdb-shipper configuration
   tsdb_shipper:
     active_index_directory: /data/tsdb-index
@@ -50,6 +51,7 @@ storage_config:
       # only applicable if using microservices where index-gateways are independently deployed.
       # This example is using kubernetes-style naming.
       server_address: dns:///index-gateway.<namespace>.svc.cluster.local:9095
+    shared_store: gcs
 
 query_scheduler:
   # the TSDB index dispatches many more, but each individually smaller, requests. 

docs/sources/setup/install/tanka.md

@@ -48,6 +48,7 @@ Revise the YAML contents of `environments/loki/main.jsonnet`, updating these var
 - Update the `username`, `password`, and the relevant `htpasswd` variable values.
 - Update the S3 or GCS variable values, depending on your object storage type. See [storage_config](/docs/loki/latest/configuration/#storage_config) for more configuration details.
 - Remove from the configuration the S3 or GCS object storage variables that are not part of your setup.
+- Update the value of `boltdb_shipper_shared_store` or `tsdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. 
 - Update the Promtail configuration `container_root_path` variable's value to reflect your root path for the Docker daemon. Run `docker info | grep "Root Dir"` to acquire your root path.
 - Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on Loki's default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range.
 
@@ -76,6 +77,9 @@ loki + promtail + gateway {
     bigtable_project: 'project',
     gcs_bucket_name: 'bucket',
 
+    //Set this variable based on the type of object storage you're using.
+    boltdb_shipper_shared_store: 'my-object-storage-backend-type',
+
     //Update the object_store and from fields
     loki+: {
       schema_config: {