Merge branch 'main' into jburnham/ruler-disable-x-scope-orgid

grafana · Feb 13, 2024 · 9da34e0 · 9da34e0
2 parents 3862ce4 + 6c5c347
commit 9da34e0
Show file tree

Hide file tree

Showing 105 changed files with 3,612 additions and 2,848 deletions.
diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
@@ -610,23 +610,6 @@ local build_image_tag = '0.33.0';
         'cd -',
       ]) { depends_on: ['clone'], when: onPRs },
       make('test', container=false) { depends_on: ['clone-target-branch', 'check-generated-files'] },
-      run('test-target-branch', commands=['cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test']) { depends_on: ['clone-target-branch'], when: onPRs },
-      make('compare-coverage', container=false, args=[
-        'old=../loki-target-branch/test_results.txt',
-        'new=test_results.txt',
-        'packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki',
-        '> diff.txt',
-      ]) { depends_on: ['test', 'test-target-branch'], when: onPRs },
-      run('report-coverage', commands=[
-        "total_diff=$(sed 's/%//' diff.txt | awk '{sum+=$3;}END{print sum;}')",
-        'if [ $total_diff = 0 ]; then exit 0; fi',
-        "pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')",
-        "body=$(jq -Rs '{body: . }' diff.txt)",
-        'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments -d "$body" > /dev/null',
-      ], env={
-        USER: 'grafanabot',
-        TOKEN: { from_secret: github_secret.name },
-      }) { depends_on: ['compare-coverage'], when: onPRs },
       make('lint', container=false) { depends_on: ['check-generated-files'] },
       make('check-mod', container=false) { depends_on: ['test', 'lint'] },
       {

diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -212,47 +212,6 @@ steps:
   environment: {}
   image: grafana/loki-build-image:0.33.0
   name: test
-- commands:
-  - cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test
-  depends_on:
-  - clone-target-branch
-  environment: {}
-  image: grafana/loki-build-image:0.33.0
-  name: test-target-branch
-  when:
-    event:
-    - pull_request
-- commands:
-  - make BUILD_IN_CONTAINER=false compare-coverage old=../loki-target-branch/test_results.txt
-    new=test_results.txt packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki
-    > diff.txt
-  depends_on:
-  - test
-  - test-target-branch
-  environment: {}
-  image: grafana/loki-build-image:0.33.0
-  name: compare-coverage
-  when:
-    event:
-    - pull_request
-- commands:
-  - total_diff=$(sed 's/%//' diff.txt | awk '{sum+=$3;}END{print sum;}')
-  - if [ $total_diff = 0 ]; then exit 0; fi
-  - pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')
-  - 'body=$(jq -Rs ''{body: . }'' diff.txt)'
-  - 'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments
-    -d "$body" > /dev/null'
-  depends_on:
-  - compare-coverage
-  environment:
-    TOKEN:
-      from_secret: github_token
-    USER: grafanabot
-  image: grafana/loki-build-image:0.33.0
-  name: report-coverage
-  when:
-    event:
-    - pull_request
 - commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
@@ -2113,6 +2072,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: 457592d17208477ceb480f81dbdb88f7b95a5ad015c88d9d6fed06c2422a52f9
+hmac: 51861919f0ba5370a152bdb9267828c742f2042819fb01388c6d23bf44e3cbb7
 
 ...
diff --git a/clients/cmd/fluent-bit/Dockerfile b/clients/cmd/fluent-bit/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.3-bullseye AS builder
+FROM golang:1.22.0-bullseye AS builder
 
 COPY . /src
 

diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md
@@ -2331,27 +2331,26 @@ bloom_shipper:
     [max_tasks_enqueued_per_tenant: <int> | default = 10000]
 
   blocks_cache:
-    # Whether embedded cache is enabled.
-    # CLI flag: -blocks-cache.enabled
+    # Cache for bloom blocks. Whether embedded cache is enabled.
+    # CLI flag: -bloom.blocks-cache.enabled
     [enabled: <boolean> | default = false]
 
-    # Maximum memory size of the cache in MB.
-    # CLI flag: -blocks-cache.max-size-mb
+    # Cache for bloom blocks. Maximum memory size of the cache in MB.
+    # CLI flag: -bloom.blocks-cache.max-size-mb
     [max_size_mb: <int> | default = 100]
 
-    # Maximum number of entries in the cache.
-    # CLI flag: -blocks-cache.max-size-items
+    # Cache for bloom blocks. Maximum number of entries in the cache.
+    # CLI flag: -bloom.blocks-cache.max-size-items
     [max_size_items: <int> | default = 0]
 
-    # The time to live for items in the cache before they get purged.
-    # CLI flag: -blocks-cache.ttl
-    [ttl: <duration> | default = 0s]
+    # Cache for bloom blocks. The time to live for items in the cache before
+    # they get purged.
+    # CLI flag: -bloom.blocks-cache.ttl
+    [ttl: <duration> | default = 24h]
 
-    # During this period the process waits until the directory becomes not used
-    # and only after this it will be deleted. If the timeout is reached, the
-    # directory is force deleted.
-    # CLI flag: -blocks-cache.remove-directory-graceful-period
-    [remove_directory_graceful_period: <duration> | default = 5m]
+  # The cache block configures the cache backend.
+  # The CLI flags prefix for this block configuration is: bloom.metas-cache
+  [metas_cache: <cache_config>]
 ```
 
 ### chunk_store_config
@@ -2646,14 +2645,27 @@ ring:
 # CLI flag: -bloom-compactor.enabled
 [enabled: <boolean> | default = false]
 
-# Directory where files can be downloaded for compaction.
-# CLI flag: -bloom-compactor.working-directory
-[working_directory: <string> | default = ""]
-
 # Interval at which to re-run the compaction operation.
 # CLI flag: -bloom-compactor.compaction-interval
 [compaction_interval: <duration> | default = 10m]
 
+# How many index periods (days) to wait before compacting a table. This can be
+# used to lower cost by not re-writing data to object storage too frequently
+# since recent data changes more often.
+# CLI flag: -bloom-compactor.min-table-compaction-period
+[min_table_compaction_period: <int> | default = 1]
+
+# How many index periods (days) to wait before compacting a table. This can be
+# used to lower cost by not trying to compact older data which doesn't change.
+# This can be optimized by aligning it with the maximum
+# `reject_old_samples_max_age` setting of any tenant.
+# CLI flag: -bloom-compactor.max-table-compaction-period
+[max_table_compaction_period: <int> | default = 7]
+
+# Number of workers to run in parallel for compaction.
+# CLI flag: -bloom-compactor.worker-parallelism
+[worker_parallelism: <int> | default = 1]
+
 # Minimum backoff time between retries.
 # CLI flag: -bloom-compactor.compaction-retries-min-backoff
 [compaction_retries_min_backoff: <duration> | default = 10s]
@@ -3133,6 +3145,12 @@ shard_streams:
 # CLI flag: -bloom-gateway.cache-key-interval
 [bloom_gateway_cache_key_interval: <duration> | default = 15m]
 
+# The maximum bloom block size. A value of 0 sets an unlimited size. Default is
+# 200MB. The actual block size might exceed this limit since blooms will be
+# added to blocks until the block exceeds the maximum block size.
+# CLI flag: -bloom-compactor.max-block-size
+[bloom_compactor_max_block_size: <int> | default = 200MB]
+
 # Allow user to send structured metadata in push payload.
 # CLI flag: -validation.allow-structured-metadata
 [allow_structured_metadata: <boolean> | default = false]
@@ -4358,6 +4376,7 @@ The TLS configuration.
 The cache block configures the cache backend. The supported CLI flags `<prefix>` used to reference this configuration block are:
 
 - `bloom-gateway-client.cache`
+- `bloom.metas-cache`
 - `frontend`
 - `frontend.index-stats-results-cache`
 - `frontend.label-results-cache`

diff --git a/integration/cluster/cluster.go b/integration/cluster/cluster.go
@@ -84,7 +84,6 @@ bloom_gateway:
 
 bloom_compactor:
   enabled: false
-  working_directory: {{.dataPath}}/bloom-compactor
 
 compactor:
   working_directory: {{.dataPath}}/compactor

diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -1,5 +1,7 @@
 ## Main
 
+- [11920](https://github.com/grafana/loki/pull/11920) **xperimental**: Refactor handling of credentials in managed-auth mode
+- [11869](https://github.com/grafana/loki/pull/11869) **periklis**: Add support for running with Google Workload Identity
 - [11868](https://github.com/grafana/loki/pull/11868) **xperimental**: Integrate support for OpenShift-managed credentials in Azure
 - [11854](https://github.com/grafana/loki/pull/11854) **periklis**: Allow custom audience for managed-auth on STS
 - [11802](https://github.com/grafana/loki/pull/11802) **xperimental**: Add support for running with Azure Workload Identity

diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go
@@ -52,16 +52,11 @@ type OpenShiftFeatureGates struct {
 	// Dashboards enables the loki-mixin dashboards into the OpenShift Console
 	Dashboards bool `json:"dashboards,omitempty"`
 
-	// ManagedAuthEnv enabled when the operator installation is on OpenShift STS clusters.
+	// ManagedAuthEnv is true when OpenShift-functions are enabled and the operator has detected
+	// that it is running with some kind of "workload identity" (AWS STS, Azure WIF) enabled.
 	ManagedAuthEnv bool
 }
 
-// ManagedAuthEnabled returns true when OpenShift-functions are enabled and the operator has detected that it is
-// running with some kind of "workload identity" (AWS STS, Azure WIF) enabled.
-func (o *OpenShiftFeatureGates) ManagedAuthEnabled() bool {
-	return o.Enabled && o.ManagedAuthEnv
-}
-
 // FeatureGates is the supported set of all operator feature gates.
 type FeatureGates struct {
 	// ServiceMonitors enables creating a Prometheus-Operator managed ServiceMonitor

diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go
@@ -1174,6 +1174,27 @@ type LokiStackComponentStatus struct {
 	Ruler PodStatusMap `json:"ruler,omitempty"`
 }
 
+// CredentialMode represents the type of authentication used for accessing the object storage.
+//
+// +kubebuilder:validation:Enum=static;token;managed
+type CredentialMode string
+
+const (
+	// CredentialModeStatic represents the usage of static, long-lived credentials stored in a Secret.
+	// This is the default authentication mode and available for all supported object storage types.
+	CredentialModeStatic CredentialMode = "static"
+	// CredentialModeToken represents the usage of short-lived tokens retrieved from a credential source.
+	// In this mode the static configuration does not contain credentials needed for the object storage.
+	// Instead, they are generated during runtime using a service, which allows for shorter-lived credentials and
+	// much more granular control. This authentication mode is not supported for all object storage types.
+	CredentialModeToken CredentialMode = "token"
+	// CredentialModeManaged represents the usage of short-lived tokens retrieved from a credential source.
+	// This mode is similar to CredentialModeToken,but instead of having a user-configured credential source,
+	// it is configured by the environment, for example the Cloud Credential Operator in OpenShift.
+	// This mode is only supported for certain object storage types in certain runtime environments.
+	CredentialModeManaged CredentialMode = "managed"
+)
+
 // LokiStackStorageStatus defines the observed state of
 // the Loki storage configuration.
 type LokiStackStorageStatus struct {
@@ -1183,6 +1204,12 @@ type LokiStackStorageStatus struct {
 	// +optional
 	// +kubebuilder:validation:Optional
 	Schemas []ObjectStorageSchema `json:"schemas,omitempty"`
+
+	// CredentialMode contains the authentication mode used for accessing the object storage.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	CredentialMode CredentialMode `json:"credentialMode,omitempty"`
 }
 
 // LokiStackStatus defines the observed state of LokiStack

diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2024-01-31T16:48:07Z"
+    createdAt: "2024-02-12T14:48:52Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     features.operators.openshift.io/disconnected: "true"
@@ -1472,6 +1472,7 @@ spec:
           - delete
           - get
           - list
+          - update
           - watch
         - apiGroups:
           - config.openshift.io

diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml
@@ -4064,6 +4064,14 @@ spec:
                 description: Storage provides summary of all changes that have occurred
                   to the storage configuration.
                 properties:
+                  credentialMode:
+                    description: CredentialMode contains the authentication mode used
+                      for accessing the object storage.
+                    enum:
+                    - static
+                    - token
+                    - managed
+                    type: string
                   schemas:
                     description: Schemas is a list of schemas which have been applied
                       to the LokiStack.

diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2024-01-31T16:48:04Z"
+    createdAt: "2024-02-12T14:48:49Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -1452,6 +1452,7 @@ spec:
           - delete
           - get
           - list
+          - update
           - watch
         - apiGroups:
           - config.openshift.io

diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml
@@ -4064,6 +4064,14 @@ spec:
                 description: Storage provides summary of all changes that have occurred
                   to the storage configuration.
                 properties:
+                  credentialMode:
+                    description: CredentialMode contains the authentication mode used
+                      for accessing the object storage.
+                    enum:
+                    - static
+                    - token
+                    - managed
+                    type: string
                   schemas:
                     description: Schemas is a list of schemas which have been applied
                       to the LokiStack.

diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: quay.io/openshift-logging/loki-operator:0.1.0
-    createdAt: "2024-01-31T16:48:10Z"
+    createdAt: "2024-02-12T14:48:55Z"
     description: |
       The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging.
       ## Prerequisites and Requirements
@@ -1457,6 +1457,7 @@ spec:
           - delete
           - get
           - list
+          - update
           - watch
         - apiGroups:
           - config.openshift.io

diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml
@@ -4064,6 +4064,14 @@ spec:
                 description: Storage provides summary of all changes that have occurred
                   to the storage configuration.
                 properties:
+                  credentialMode:
+                    description: CredentialMode contains the authentication mode used
+                      for accessing the object storage.
+                    enum:
+                    - static
+                    - token
+                    - managed
+                    type: string
                   schemas:
                     description: Schemas is a list of schemas which have been applied
                       to the LokiStack.

diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
@@ -4046,6 +4046,14 @@ spec:
                 description: Storage provides summary of all changes that have occurred
                   to the storage configuration.
                 properties:
+                  credentialMode:
+                    description: CredentialMode contains the authentication mode used
+                      for accessing the object storage.
+                    enum:
+                    - static
+                    - token
+                    - managed
+                    type: string
                   schemas:
                     description: Schemas is a list of schemas which have been applied
                       to the LokiStack.

diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml
@@ -56,6 +56,7 @@ rules:
   - delete
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - config.openshift.io
-Original file line number
+Diff line change
@@ Expand Up / @@ -56,6 +56,7 @@ rules: @@
       - delete
       - get
       - list
+      - update
       - watch
     - apiGroups:
       - config.openshift.io
@@ Expand Down @@