operator: Add support for running with Google Workload Identity (#11869)

Co-authored-by: Robert Jacob <[email protected]>

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [11869](https://github.com/grafana/loki/pull/11869) **periklis**: Add support for running with Google Workload Identity
 - [11868](https://github.com/grafana/loki/pull/11868) **xperimental**: Integrate support for OpenShift-managed credentials in Azure
 - [11854](https://github.com/grafana/loki/pull/11854) **periklis**: Allow custom audience for managed-auth on STS
 - [11802](https://github.com/grafana/loki/pull/11802) **xperimental**: Add support for running with Azure Workload Identity

operator/internal/handlers/internal/storage/secrets.go

@@ -3,6 +3,7 @@ package storage
 import (
 	"context"
 	"crypto/sha1"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"sort"
@@ -32,8 +33,13 @@ var (
 	errAzureNoCredentials             = errors.New("azure storage secret does contain neither account_key or client_id")
 	errAzureMixedCredentials          = errors.New("azure storage secret can not contain both account_key and client_id")
 	errAzureManagedIdentityNoOverride = errors.New("when in managed mode, storage secret can not contain credentials")
+
+	errGCPParseCredentialsFile      = errors.New("gcp storage secret cannot be parsed from JSON content")
+	errGCPWrongCredentialSourceFile = errors.New("credential source in secret needs to point to token file")
 )
 
+const gcpAccountTypeExternal = "external_account"
+
 func getSecrets(ctx context.Context, k k8s.Client, stack *lokiv1.LokiStack, fg configv1.FeatureGates) (*corev1.Secret, *corev1.Secret, error) {
 	var (
 		storageSecret     corev1.Secret
@@ -255,8 +261,36 @@ func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error)
 		return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyGCPServiceAccountKeyFilename)
 	}
 
+	credentialsFile := struct {
+		CredentialsType   string `json:"type"`
+		CredentialsSource struct {
+			File string `json:"file"`
+		} `json:"credential_source"`
+	}{}
+
+	err := json.Unmarshal(keyJSON, &credentialsFile)
+	if err != nil {
+		return nil, errGCPParseCredentialsFile
+	}
+
+	var (
+		audience           = s.Data[storage.KeyGCPWorkloadIdentityProviderAudience]
+		isWorkloadIdentity = credentialsFile.CredentialsType == gcpAccountTypeExternal
+	)
+	if isWorkloadIdentity {
+		if len(audience) == 0 {
+			return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyGCPWorkloadIdentityProviderAudience)
+		}
+
+		if credentialsFile.CredentialsSource.File != storage.GCPDefautCredentialsFile {
+			return nil, fmt.Errorf("%w: %s", errGCPWrongCredentialSourceFile, storage.GCPDefautCredentialsFile)
+		}
+	}
+
 	return &storage.GCSStorageConfig{
-		Bucket: string(bucket),
+		Bucket:           string(bucket),
+		WorkloadIdentity: isWorkloadIdentity,
+		Audience:         string(audience),
 	}, nil
 }
 

operator/internal/handlers/internal/storage/secrets_test.go

@@ -314,13 +314,47 @@ func TestGCSExtract(t *testing.T) {
 			},
 			wantError: "missing secret field: key.json",
 		},
+		{
+			name: "missing audience",
+			secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: "test"},
+				Data: map[string][]byte{
+					"bucketname": []byte("here"),
+					"key.json":   []byte("{\"type\": \"external_account\"}"),
+				},
+			},
+			wantError: "missing secret field: audience",
+		},
+		{
+			name: "credential_source file no override",
+			secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: "test"},
+				Data: map[string][]byte{
+					"bucketname": []byte("here"),
+					"audience":   []byte("test"),
+					"key.json":   []byte("{\"type\": \"external_account\", \"credential_source\": {\"file\": \"/custom/path/to/secret/gcp/serviceaccount/token\"}}"),
+				},
+			},
+			wantError: "credential source in secret needs to point to token file: /var/run/secrets/gcp/serviceaccount/token",
+		},
 		{
 			name: "all set",
 			secret: &corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{Name: "test"},
 				Data: map[string][]byte{
 					"bucketname": []byte("here"),
-					"key.json":   []byte("{\"type\": \"SA\"}"),
+					"key.json":   []byte("{\"type\": \"service_account\"}"),
+				},
+			},
+		},
+		{
+			name: "mandatory for workload-identity set",
+			secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{Name: "test"},
+				Data: map[string][]byte{
+					"bucketname": []byte("here"),
+					"audience":   []byte("test"),
+					"key.json":   []byte("{\"type\": \"external_account\", \"credential_source\": {\"file\": \"/var/run/secrets/gcp/serviceaccount/token\"}}"),
 				},
 			},
 		},

operator/internal/manifests/storage/configure.go

@@ -210,6 +210,10 @@ func managedAuthCredentials(opts Options) []corev1.EnvVar {
 			envVarFromSecret(EnvAzureSubscriptionID, opts.SecretName, KeyAzureStorageSubscriptionID),
 			envVarFromValue(EnvAzureFederatedTokenFile, path.Join(azureTokenVolumeDirectory, "token")),
 		}
+	case lokiv1.ObjectStorageSecretGCS:
+		return []corev1.EnvVar{
+			envVarFromValue(EnvGoogleApplicationCredentials, path.Join(secretDirectory, KeyGCPServiceAccountKeyFilename)),
+		}
 	default:
 		return []corev1.EnvVar{}
 	}
@@ -290,6 +294,8 @@ func managedAuthEnabled(opts Options) bool {
 		return opts.S3 != nil && opts.S3.STS
 	case lokiv1.ObjectStorageSecretAzure:
 		return opts.Azure != nil && opts.Azure.WorkloadIdentity
+	case lokiv1.ObjectStorageSecretGCS:
+		return opts.GCS != nil && opts.GCS.WorkloadIdentity
 	default:
 		return false
 	}
@@ -302,6 +308,8 @@ func saTokenVolumeMount(opts Options) corev1.VolumeMount {
 		tokenPath = AWSTokenVolumeDirectory
 	case lokiv1.ObjectStorageSecretAzure:
 		tokenPath = azureTokenVolumeDirectory
+	case lokiv1.ObjectStorageSecretGCS:
+		tokenPath = gcpTokenVolumeDirectory
 	}
 	return corev1.VolumeMount{
 		Name:      saTokenVolumeName,
@@ -323,6 +331,8 @@ func saTokenVolume(opts Options) corev1.Volume {
 		if opts.Azure.Audience != "" {
 			audience = opts.Azure.Audience
 		}
+	case lokiv1.ObjectStorageSecretGCS:
+		audience = opts.GCS.Audience
 	}
 	return corev1.Volume{
 		Name: saTokenVolumeName,

operator/internal/manifests/storage/configure_test.go

@@ -616,6 +616,87 @@ func TestConfigureDeploymentForStorageType(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "object storage GCS with Workload Identity",
+			opts: Options{
+				SecretName:  "test",
+				SharedStore: lokiv1.ObjectStorageSecretGCS,
+				GCS: &GCSStorageConfig{
+					Audience:         "test",
+					WorkloadIdentity: true,
+				},
+			},
+			dpl: &appsv1.Deployment{
+				Spec: appsv1.DeploymentSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "loki-ingester",
+								},
+							},
+						},
+					},
+				},
+			},
+			want: &appsv1.Deployment{
+				Spec: appsv1.DeploymentSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "loki-ingester",
+									VolumeMounts: []corev1.VolumeMount{
+										{
+											Name:      "test",
+											ReadOnly:  false,
+											MountPath: "/etc/storage/secrets",
+										},
+										{
+											Name:      saTokenVolumeName,
+											ReadOnly:  false,
+											MountPath: "/var/run/secrets/gcp/serviceaccount",
+										},
+									},
+									Env: []corev1.EnvVar{
+										{
+											Name:  EnvGoogleApplicationCredentials,
+											Value: "/etc/storage/secrets/key.json",
+										},
+									},
+								},
+							},
+							Volumes: []corev1.Volume{
+								{
+									Name: "test",
+									VolumeSource: corev1.VolumeSource{
+										Secret: &corev1.SecretVolumeSource{
+											SecretName: "test",
+										},
+									},
+								},
+								{
+									Name: saTokenVolumeName,
+									VolumeSource: corev1.VolumeSource{
+										Projected: &corev1.ProjectedVolumeSource{
+											Sources: []corev1.VolumeProjection{
+												{
+													ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
+														Audience:          "test",
+														ExpirationSeconds: ptr.To[int64](3600),
+														Path:              corev1.ServiceAccountTokenKey,
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 		{
 			desc: "object storage S3",
 			opts: Options{
@@ -1669,6 +1750,87 @@ func TestConfigureStatefulSetForStorageType(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "object storage GCS with Workload Identity",
+			opts: Options{
+				SecretName:  "test",
+				SharedStore: lokiv1.ObjectStorageSecretGCS,
+				GCS: &GCSStorageConfig{
+					Audience:         "test",
+					WorkloadIdentity: true,
+				},
+			},
+			sts: &appsv1.StatefulSet{
+				Spec: appsv1.StatefulSetSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "loki-ingester",
+								},
+							},
+						},
+					},
+				},
+			},
+			want: &appsv1.StatefulSet{
+				Spec: appsv1.StatefulSetSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "loki-ingester",
+									VolumeMounts: []corev1.VolumeMount{
+										{
+											Name:      "test",
+											ReadOnly:  false,
+											MountPath: "/etc/storage/secrets",
+										},
+										{
+											Name:      saTokenVolumeName,
+											ReadOnly:  false,
+											MountPath: "/var/run/secrets/gcp/serviceaccount",
+										},
+									},
+									Env: []corev1.EnvVar{
+										{
+											Name:  EnvGoogleApplicationCredentials,
+											Value: "/etc/storage/secrets/key.json",
+										},
+									},
+								},
+							},
+							Volumes: []corev1.Volume{
+								{
+									Name: "test",
+									VolumeSource: corev1.VolumeSource{
+										Secret: &corev1.SecretVolumeSource{
+											SecretName: "test",
+										},
+									},
+								},
+								{
+									Name: saTokenVolumeName,
+									VolumeSource: corev1.VolumeSource{
+										Projected: &corev1.ProjectedVolumeSource{
+											Sources: []corev1.VolumeProjection{
+												{
+													ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
+														Audience:          "test",
+														ExpirationSeconds: ptr.To[int64](3600),
+														Path:              corev1.ServiceAccountTokenKey,
+													},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 		{
 			desc: "object storage S3",
 			opts: Options{

operator/internal/manifests/storage/options.go

@@ -34,7 +34,9 @@ type AzureStorageConfig struct {
 
 // GCSStorageConfig for GCS storage config
 type GCSStorageConfig struct {
-	Bucket string
+	Bucket           string
+	Audience         string
+	WorkloadIdentity bool
 }
 
 // S3StorageConfig for S3 storage config

operator/internal/manifests/storage/var.go

@@ -91,6 +91,8 @@ const (
 	// KeyAzureAudience is the secret data key for customizing the audience used for the ServiceAccount token.
 	KeyAzureAudience = "audience"
 
+	// KeyGCPWorkloadIdentityProviderAudience is the secret data key for the GCP Workload Identity Provider audience.
+	KeyGCPWorkloadIdentityProviderAudience = "audience"
 	// KeyGCPStorageBucketName is the secret data key for the GCS bucket name.
 	KeyGCPStorageBucketName = "bucketname"
 	// KeyGCPServiceAccountKeyFilename is the service account key filename containing the Google authentication credentials.
@@ -144,5 +146,8 @@ const (
 	azureManagedCredentialKeyTenantID       = "azure_tenant_id"
 	azureManagedCredentialKeySubscriptionID = "azure_subscription_id"
 
+	gcpTokenVolumeDirectory  = "/var/run/secrets/gcp/serviceaccount"
+	GCPDefautCredentialsFile = gcpTokenVolumeDirectory + "/token"
+
 	AnnotationCredentialsRequestsSecretRef = "loki.grafana.com/credentials-request-secret-ref"
 )