Merge branch 'main' into fix-netpols

grafana · Feb 12, 2024 · e100eab · e100eab
2 parents 337acd7 + 7a95cb8
commit e100eab
Show file tree

Hide file tree

Showing 82 changed files with 4,189 additions and 2,280 deletions.
diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
@@ -610,23 +610,6 @@ local build_image_tag = '0.33.0';
         'cd -',
       ]) { depends_on: ['clone'], when: onPRs },
       make('test', container=false) { depends_on: ['clone-target-branch', 'check-generated-files'] },
-      run('test-target-branch', commands=['cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test']) { depends_on: ['clone-target-branch'], when: onPRs },
-      make('compare-coverage', container=false, args=[
-        'old=../loki-target-branch/test_results.txt',
-        'new=test_results.txt',
-        'packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki',
-        '> diff.txt',
-      ]) { depends_on: ['test', 'test-target-branch'], when: onPRs },
-      run('report-coverage', commands=[
-        "total_diff=$(sed 's/%//' diff.txt | awk '{sum+=$3;}END{print sum;}')",
-        'if [ $total_diff = 0 ]; then exit 0; fi',
-        "pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')",
-        "body=$(jq -Rs '{body: . }' diff.txt)",
-        'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments -d "$body" > /dev/null',
-      ], env={
-        USER: 'grafanabot',
-        TOKEN: { from_secret: github_secret.name },
-      }) { depends_on: ['compare-coverage'], when: onPRs },
       make('lint', container=false) { depends_on: ['check-generated-files'] },
       make('check-mod', container=false) { depends_on: ['test', 'lint'] },
       {

diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -212,47 +212,6 @@ steps:
   environment: {}
   image: grafana/loki-build-image:0.33.0
   name: test
-- commands:
-  - cd ../loki-target-branch && BUILD_IN_CONTAINER=false make test
-  depends_on:
-  - clone-target-branch
-  environment: {}
-  image: grafana/loki-build-image:0.33.0
-  name: test-target-branch
-  when:
-    event:
-    - pull_request
-- commands:
-  - make BUILD_IN_CONTAINER=false compare-coverage old=../loki-target-branch/test_results.txt
-    new=test_results.txt packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki
-    > diff.txt
-  depends_on:
-  - test
-  - test-target-branch
-  environment: {}
-  image: grafana/loki-build-image:0.33.0
-  name: compare-coverage
-  when:
-    event:
-    - pull_request
-- commands:
-  - total_diff=$(sed 's/%//' diff.txt | awk '{sum+=$3;}END{print sum;}')
-  - if [ $total_diff = 0 ]; then exit 0; fi
-  - pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')
-  - 'body=$(jq -Rs ''{body: . }'' diff.txt)'
-  - 'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments
-    -d "$body" > /dev/null'
-  depends_on:
-  - compare-coverage
-  environment:
-    TOKEN:
-      from_secret: github_token
-    USER: grafanabot
-  image: grafana/loki-build-image:0.33.0
-  name: report-coverage
-  when:
-    event:
-    - pull_request
 - commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
@@ -2113,6 +2072,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: 457592d17208477ceb480f81dbdb88f7b95a5ad015c88d9d6fed06c2422a52f9
+hmac: 51861919f0ba5370a152bdb9267828c742f2042819fb01388c6d23bf44e3cbb7
 
 ...
diff --git a/clients/cmd/fluent-bit/Dockerfile b/clients/cmd/fluent-bit/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.21.3-bullseye AS builder
+FROM golang:1.22.0-bullseye AS builder
 
 COPY . /src
 

diff --git a/docs/sources/configure/_index.md b/docs/sources/configure/_index.md
@@ -2327,27 +2327,26 @@ bloom_shipper:
     [max_tasks_enqueued_per_tenant: <int> | default = 10000]
 
   blocks_cache:
-    # Whether embedded cache is enabled.
-    # CLI flag: -blocks-cache.enabled
+    # Cache for bloom blocks. Whether embedded cache is enabled.
+    # CLI flag: -bloom.blocks-cache.enabled
     [enabled: <boolean> | default = false]
 
-    # Maximum memory size of the cache in MB.
-    # CLI flag: -blocks-cache.max-size-mb
+    # Cache for bloom blocks. Maximum memory size of the cache in MB.
+    # CLI flag: -bloom.blocks-cache.max-size-mb
     [max_size_mb: <int> | default = 100]
 
-    # Maximum number of entries in the cache.
-    # CLI flag: -blocks-cache.max-size-items
+    # Cache for bloom blocks. Maximum number of entries in the cache.
+    # CLI flag: -bloom.blocks-cache.max-size-items
     [max_size_items: <int> | default = 0]
 
-    # The time to live for items in the cache before they get purged.
-    # CLI flag: -blocks-cache.ttl
-    [ttl: <duration> | default = 0s]
+    # Cache for bloom blocks. The time to live for items in the cache before
+    # they get purged.
+    # CLI flag: -bloom.blocks-cache.ttl
+    [ttl: <duration> | default = 24h]
 
-    # During this period the process waits until the directory becomes not used
-    # and only after this it will be deleted. If the timeout is reached, the
-    # directory is force deleted.
-    # CLI flag: -blocks-cache.remove-directory-graceful-period
-    [remove_directory_graceful_period: <duration> | default = 5m]
+  # The cache block configures the cache backend.
+  # The CLI flags prefix for this block configuration is: bloom.metas-cache
+  [metas_cache: <cache_config>]
 ```
 
 ### chunk_store_config
@@ -2650,6 +2649,18 @@ ring:
 # CLI flag: -bloom-compactor.compaction-interval
 [compaction_interval: <duration> | default = 10m]
 
+# Minimum age of a table before it is considered for compaction.
+# CLI flag: -bloom-compactor.min-compaction-age
+[min_compaction_age: <duration> | default = 24h]
+
+# Maximum age of a table before it is considered for compaction.
+# CLI flag: -bloom-compactor.max-compaction-age
+[max_compaction_age: <duration> | default = 168h]
+
+# Number of workers to run in parallel for compaction.
+# CLI flag: -bloom-compactor.worker-parallelism
+[worker_parallelism: <int> | default = 1]
+
 # Minimum backoff time between retries.
 # CLI flag: -bloom-compactor.compaction-retries-min-backoff
 [compaction_retries_min_backoff: <duration> | default = 10s]
@@ -3129,6 +3140,12 @@ shard_streams:
 # CLI flag: -bloom-gateway.cache-key-interval
 [bloom_gateway_cache_key_interval: <duration> | default = 15m]
 
+# The maximum bloom block size. A value of 0 sets an unlimited size. Default is
+# 200MB. The actual block size might exceed this limit since blooms will be
+# added to blocks until the block exceeds the maximum block size.
+# CLI flag: -bloom-compactor.max-block-size
+[bloom_compactor_max_block_size: <int> | default = 200MB]
+
 # Allow user to send structured metadata in push payload.
 # CLI flag: -validation.allow-structured-metadata
 [allow_structured_metadata: <boolean> | default = false]
@@ -4354,6 +4371,7 @@ The TLS configuration.
 The cache block configures the cache backend. The supported CLI flags `<prefix>` used to reference this configuration block are:
 
 - `bloom-gateway-client.cache`
+- `bloom.metas-cache`
 - `frontend`
 - `frontend.index-stats-results-cache`
 - `frontend.label-results-cache`

diff --git a/docs/sources/get-started/labels/structured-metadata.md b/docs/sources/get-started/labels/structured-metadata.md
@@ -5,10 +5,6 @@ description: Describes how to enable structure metadata for logs and how to quer
 ---
 # What is structured metadata
 
-{{% admonition type="warning" %}}
-Structured metadata is an experimental feature and is subject to change in future releases of Grafana Loki. This feature is not yet available for Cloud Logs users.
-{{% /admonition %}}
-
 {{% admonition type="warning" %}}
 Structured metadata was added to chunk format V4 which is used if the schema version is greater or equal to `13`. (See [Schema Config]({{< relref "../../storage#schema-config" >}}) for more details about schema versions. )
 {{% /admonition %}}

diff --git a/docs/sources/operations/query-fairness/_index.md b/docs/sources/operations/query-fairness/_index.md
@@ -115,7 +115,7 @@ you would usually want to avoid this scenario and control yourself where the hea
 
 When using Grafana as the Loki user interface, you can, for example, create multiple data sources
 with the same tenant, but with a different additional HTTP header
-`X-Loki-Scope-Actor` and restrict which Grafana user can use which data source.
+`X-Loki-Actor-Path` and restrict which Grafana user can use which data source.
 
 Alternatively, if you have a proxy for authentication in front of Loki, you can
 pass the (hashed) user from the authentication as downstream header to Loki.
diff --git a/docs/sources/release-notes/v2-9.md b/docs/sources/release-notes/v2-9.md
@@ -9,6 +9,8 @@ Grafana Labs is excited to announce the release of Loki 2.9.0 Here's a summary o
 
 ## Features and enhancements
 
+- **Structured metadata**: The [Structured Metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/) feature, which was introduced as experimental in release 2.9.0, is generally available as of release 2.9.4.
+
 -  **Query Language Improvements**: Several improvements to the query language that speed up line parsing and regex matching. [PR #8646](https://github.com/grafana/loki/pull/8646), [PR #8659](https://github.com/grafana/loki/pull/8659), [PR #8724](https://github.com/grafana/loki/pull/8724), [PR #8734](https://github.com/grafana/loki/pull/8734), [PR #8739](https://github.com/grafana/loki/pull/8739), [PR #8763](https://github.com/grafana/loki/pull/8763), [PR #8890](https://github.com/grafana/loki/pull/8890), [PR #8914](https://github.com/grafana/loki/pull/8914)
 
 -  **Remote rule evaluation**: Rule evaluation can now be handled by queriers to improve speed. [PR #8744](https://github.com/grafana/loki/pull/8744) [PR #8848](https://github.com/grafana/loki/pull/8848)
@@ -33,13 +35,13 @@ Grafana Labs is excited to announce the release of Loki 2.9.0 Here's a summary o
 
 ## Bug fixes
 
-### 2.9.1 (2023-09-14)
-
-* Update Docker base images to mitigate security vulnerability CVE-2022-48174
-* Fix bugs in indexshipper (`tsdb`, `boltdb-shipper`) that could result in not showing all ingested logs in query results.
-
 ### 2.9.2 (2023-10-16)
 
 * Upgrade go to v1.21.3, golang.org/x/net to v0.17.0 and grpc-go to v1.56.3 to patch CVE-2023-39325 / CVE-2023-44487
 
 For a full list of all changes and fixes, look at the [CHANGELOG](https://github.com/grafana/loki/blob/release-2.9.x/CHANGELOG.md).
+
+### 2.9.1 (2023-09-14)
+
+* Update Docker base images to mitigate security vulnerability CVE-2022-48174
+* Fix bugs in indexshipper (`tsdb`, `boltdb-shipper`) that could result in not showing all ingested logs in query results.
diff --git a/docs/sources/send-data/otel/_index.md b/docs/sources/send-data/otel/_index.md
@@ -9,10 +9,6 @@ weight:  250
 
 # Ingesting logs to Loki using OpenTelemetry Collector
 
-{{% admonition type="warning" %}}
-OpenTelemetry logs ingestion is an experimental feature and is subject to change in future releases of Grafana Loki.
-{{% /admonition %}}
-
 Loki natively supports ingesting OpenTelemetry logs over HTTP.
 For ingesting logs to Loki using the OpenTelemetry Collector, you must use the [`otlphttp` exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter).
 

diff --git a/docs/sources/setup/install/docker.md b/docs/sources/setup/install/docker.md
@@ -9,7 +9,7 @@ weight: 400
 # Install Loki with Docker or Docker Compose
 
 You can install Loki and Promtail with Docker or Docker Compose if you are evaluating, testing, or developing Loki.
-For production, we recommend installing with Tanka or Helm.
+For production, Grafana recommends installing with Tanka or Helm.
 
 The configuration acquired with these installation instructions run Loki as a single binary.
 

diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -1,5 +1,7 @@
 ## Main
 
+- [11869](https://github.com/grafana/loki/pull/11869) **periklis**: Add support for running with Google Workload Identity
+- [11868](https://github.com/grafana/loki/pull/11868) **xperimental**: Integrate support for OpenShift-managed credentials in Azure
 - [11854](https://github.com/grafana/loki/pull/11854) **periklis**: Allow custom audience for managed-auth on STS
 - [11802](https://github.com/grafana/loki/pull/11802) **xperimental**: Add support for running with Azure Workload Identity
 - [11824](https://github.com/grafana/loki/pull/11824) **xperimental**: Improve messages for errors in storage secret

diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go
@@ -56,7 +56,9 @@ type OpenShiftFeatureGates struct {
 	ManagedAuthEnv bool
 }
 
-func (o OpenShiftFeatureGates) ManagedAuthEnabled() bool {
+// ManagedAuthEnabled returns true when OpenShift-functions are enabled and the operator has detected that it is
+// running with some kind of "workload identity" (AWS STS, Azure WIF) enabled.
+func (o *OpenShiftFeatureGates) ManagedAuthEnabled() bool {
 	return o.Enabled && o.ManagedAuthEnv
 }
 

diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2024-01-25T11:08:43Z"
+    createdAt: "2024-01-31T16:48:07Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     features.operators.openshift.io/disconnected: "true"

diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.5.0
-    createdAt: "2024-01-25T11:08:41Z"
+    createdAt: "2024-01-31T16:48:04Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown

diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: quay.io/openshift-logging/loki-operator:0.1.0
-    createdAt: "2024-01-25T11:08:45Z"
+    createdAt: "2024-01-31T16:48:10Z"
     description: |
       The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging.
       ## Prerequisites and Requirements
@@ -165,7 +165,7 @@ metadata:
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
     features.operators.openshift.io/token-auth-aws: "true"
-    features.operators.openshift.io/token-auth-azure: "false"
+    features.operators.openshift.io/token-auth-azure: "true"
     features.operators.openshift.io/token-auth-gcp: "false"
     olm.skipRange: '>=5.7.0-0 <5.9.0'
     operatorframework.io/cluster-monitoring: "true"

diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml
@@ -21,7 +21,7 @@ metadata:
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
     features.operators.openshift.io/token-auth-aws: "true"
-    features.operators.openshift.io/token-auth-azure: "false"
+    features.operators.openshift.io/token-auth-azure: "true"
     features.operators.openshift.io/token-auth-gcp: "false"
     olm.skipRange: '>=5.7.0-0 <5.9.0'
     operatorframework.io/cluster-monitoring: "true"

diff --git a/operator/controllers/loki/credentialsrequests_controller.go b/operator/controllers/loki/credentialsrequests_controller.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -46,7 +47,17 @@ func (r *CredentialsRequestsReconciler) Reconcile(ctx context.Context, req ctrl.
 		return ctrl.Result{}, nil
 	}
 
-	secretRef, err := handlers.CreateCredentialsRequest(ctx, r.Client, req.NamespacedName)
+	storageSecretName := client.ObjectKey{
+		Namespace: req.Namespace,
+		Name:      stack.Spec.Storage.Secret.Name,
+	}
+	storageSecret := &corev1.Secret{}
+	err = r.Client.Get(ctx, storageSecretName, storageSecret)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	secretRef, err := handlers.CreateCredentialsRequest(ctx, r.Client, req.NamespacedName, storageSecret)
 	if err != nil {
 		return ctrl.Result{}, err
 	}

diff --git a/operator/controllers/loki/credentialsrequests_controller_test.go b/operator/controllers/loki/credentialsrequests_controller_test.go
@@ -6,6 +6,7 @@ import (
 
 	cloudcredentialsv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1"
 	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -81,16 +82,24 @@ func TestCredentialsRequestController_CreateCredentialsRequest_WhenLokiStackNotA
 			ManagementState: lokiv1.ManagementStateManaged,
 		},
 	}
+	secret := &corev1.Secret{}
 
 	// Set managed auth environment
 	t.Setenv("ROLEARN", "a-role-arn")
 
 	k.GetStub = func(_ context.Context, key types.NamespacedName, out client.Object, _ ...client.GetOption) error {
-		if key.Name == r.Name && key.Namespace == r.Namespace {
-			k.SetClientObject(out, &s)
+		switch out.(type) {
+		case *lokiv1.LokiStack:
+			if key.Name == r.Name && key.Namespace == r.Namespace {
+				k.SetClientObject(out, &s)
+				return nil
+			}
+			return apierrors.NewNotFound(schema.GroupResource{}, "lokistack not found")
+		case *corev1.Secret:
+			k.SetClientObject(out, secret)
 			return nil
 		}
-		return apierrors.NewNotFound(schema.GroupResource{}, "lokistack not found")
+		return nil
 	}
 
 	k.CreateStub = func(_ context.Context, o client.Object, _ ...client.CreateOption) error {