From 6fdc7e2ed0ff3af272e68ba6b79844452dbc8c26 Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Mon, 4 Nov 2024 15:55:53 +0100 Subject: [PATCH] feat(operator): Add support for managed GCP WorkloadIdentity --- operator/internal/config/managed_auth.go | 17 ++++ .../handlers/internal/storage/secrets.go | 43 +++++--- .../handlers/internal/storage/secrets_test.go | 43 +++++++- .../manifests/openshift/credentialsrequest.go | 9 ++ .../internal/manifests/storage/configure.go | 14 ++- .../manifests/storage/configure_test.go | 98 +++++++++++++++++++ operator/internal/manifests/storage/var.go | 2 + 7 files changed, 207 insertions(+), 19 deletions(-) diff --git a/operator/internal/config/managed_auth.go b/operator/internal/config/managed_auth.go index 6e3dc524716b1..c2959f7f82e96 100644 --- a/operator/internal/config/managed_auth.go +++ b/operator/internal/config/managed_auth.go @@ -13,9 +13,15 @@ type AzureEnvironment struct { Region string } +type GCPEnvironment struct { + Audience string + ServiceAccountEmail string +} + type TokenCCOAuthConfig struct { AWS *AWSEnvironment Azure *AzureEnvironment + GCP *GCPEnvironment } func discoverTokenCCOAuthConfig() *TokenCCOAuthConfig { @@ -28,6 +34,10 @@ func discoverTokenCCOAuthConfig() *TokenCCOAuthConfig { subscriptionID := os.Getenv("SUBSCRIPTIONID") region := os.Getenv("REGION") + // GCP + audience := os.Getenv("AUDIENCE") + serviceAccountEmail := os.Getenv("SERVICE_ACCOUNT_EMAIL") + switch { case roleARN != "": return &TokenCCOAuthConfig{ @@ -44,6 +54,13 @@ func discoverTokenCCOAuthConfig() *TokenCCOAuthConfig { Region: region, }, } + case audience != "" && serviceAccountEmail != "": + return &TokenCCOAuthConfig{ + GCP: &GCPEnvironment{ + Audience: audience, + ServiceAccountEmail: serviceAccountEmail, + }, + } } return nil diff --git a/operator/internal/handlers/internal/storage/secrets.go b/operator/internal/handlers/internal/storage/secrets.go index 36065afb4c8ab..744d1abd370b3 100644 --- a/operator/internal/handlers/internal/storage/secrets.go +++ b/operator/internal/handlers/internal/storage/secrets.go @@ -33,8 +33,7 @@ var ( errSecretUnknownSSEType = errors.New("unsupported SSE type (supported: SSE-KMS, SSE-S3)") errSecretHashError = errors.New("error calculating hash for secret") - errSecretUnknownCredentialMode = errors.New("unknown credential mode") - errSecretUnsupportedCredentialMode = errors.New("combination of storage type and credential mode not supported") + errSecretUnknownCredentialMode = errors.New("unknown credential mode") errAzureManagedIdentityNoOverride = errors.New("when in managed mode, storage secret can not contain credentials") errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)") @@ -47,6 +46,7 @@ var ( errGCPParseCredentialsFile = errors.New("gcp storage secret cannot be parsed from JSON content") errGCPWrongCredentialSourceFile = errors.New("credential source in secret needs to point to token file") + errGCPInvalidCredentialsFile = errors.New("GCP credentials file contains invalid fields") azureValidEnvironments = map[string]bool{ "AzureGlobal": true, @@ -138,7 +138,7 @@ func extractSecrets(secretSpec lokiv1.ObjectStorageSecretSpec, objStore, tokenCC case lokiv1.ObjectStorageSecretAzure: storageOpts.Azure, err = extractAzureConfigSecret(objStore, credentialMode) case lokiv1.ObjectStorageSecretGCS: - storageOpts.GCS, err = extractGCSConfigSecret(objStore, credentialMode) + storageOpts.GCS, err = extractGCSConfigSecret(objStore, tokenCCOAuth, credentialMode) case lokiv1.ObjectStorageSecretS3: storageOpts.S3, err = extractS3ConfigSecret(objStore, credentialMode) case lokiv1.ObjectStorageSecretSwift: @@ -182,7 +182,7 @@ func determineCredentialMode(spec lokiv1.ObjectStorageSecretSpec, secret *corev1 return lokiv1.CredentialModeToken, nil } case lokiv1.ObjectStorageSecretGCS: - _, credentialType, err := extractGoogleCredentialSource(secret) + _, credentialType, _, err := extractGoogleCredentialSource(secret, storage.KeyGCPServiceAccountKeyFilename) if err != nil { return "", err } @@ -326,13 +326,14 @@ func validateBase64(data []byte) error { return err } -func extractGoogleCredentialSource(secret *corev1.Secret) (sourceFile, sourceType string, err error) { - keyJSON := secret.Data[storage.KeyGCPServiceAccountKeyFilename] +func extractGoogleCredentialSource(secret *corev1.Secret, key string) (sourceFile, sourceType, audience string, err error) { + keyJSON := secret.Data[key] if len(keyJSON) == 0 { - return "", "", fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyGCPServiceAccountKeyFilename) + return "", "", "", fmt.Errorf("%w: %s", errSecretMissingField, key) } credentialsFile := struct { + Audience string `json:"audience"` CredentialsType string `json:"type"` CredentialsSource struct { File string `json:"file"` @@ -341,13 +342,13 @@ func extractGoogleCredentialSource(secret *corev1.Secret) (sourceFile, sourceTyp err = json.Unmarshal(keyJSON, &credentialsFile) if err != nil { - return "", "", errGCPParseCredentialsFile + return "", "", "", errGCPParseCredentialsFile } - return credentialsFile.CredentialsSource.File, credentialsFile.CredentialsType, nil + return credentialsFile.CredentialsSource.File, credentialsFile.CredentialsType, credentialsFile.Audience, nil } -func extractGCSConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMode) (*storage.GCSStorageConfig, error) { +func extractGCSConfigSecret(s, tokenCCOAuth *corev1.Secret, credentialMode lokiv1.CredentialMode) (*storage.GCSStorageConfig, error) { // Extract and validate mandatory fields bucket := s.Data[storage.KeyGCPStorageBucketName] if len(bucket) == 0 { @@ -355,6 +356,21 @@ func extractGCSConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMo } switch credentialMode { + case lokiv1.CredentialModeTokenCCO: + if _, ok := s.Data[storage.KeyGCPServiceAccountKeyFilename]; ok { + return nil, fmt.Errorf("%w: %s", errGCPInvalidCredentialsFile, "key.json must not be set for CredentialModeTokenCCO") + } + + _, _, audience, err := extractGoogleCredentialSource(tokenCCOAuth, storage.KeyGCPManagedServiceAccountKeyFilename) + if err != nil { + return nil, err + } + + return &storage.GCSStorageConfig{ + Audience: audience, + Bucket: string(bucket), + WorkloadIdentity: true, + }, nil case lokiv1.CredentialModeStatic: return &storage.GCSStorageConfig{ Bucket: string(bucket), @@ -366,7 +382,7 @@ func extractGCSConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMo } // Check if correct credential source is used - credentialSource, _, err := extractGoogleCredentialSource(s) + credentialSource, _, _, err := extractGoogleCredentialSource(s, storage.KeyGCPServiceAccountKeyFilename) if err != nil { return nil, err } @@ -380,12 +396,9 @@ func extractGCSConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMo WorkloadIdentity: true, Audience: audience, }, nil - case lokiv1.CredentialModeTokenCCO: - return nil, fmt.Errorf("%w: type: %s credentialMode: %s", errSecretUnsupportedCredentialMode, lokiv1.ObjectStorageSecretGCS, credentialMode) default: + return nil, fmt.Errorf("%w: %s", errSecretUnknownCredentialMode, credentialMode) } - - return nil, fmt.Errorf("%w: %s", errSecretUnknownCredentialMode, credentialMode) } func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMode) (*storage.S3StorageConfig, error) { diff --git a/operator/internal/handlers/internal/storage/secrets_test.go b/operator/internal/handlers/internal/storage/secrets_test.go index 0688b099f3a86..3d63fcc177e3f 100644 --- a/operator/internal/handlers/internal/storage/secrets_test.go +++ b/operator/internal/handlers/internal/storage/secrets_test.go @@ -277,6 +277,8 @@ func TestGCSExtract(t *testing.T) { type test struct { name string secret *corev1.Secret + tokenAuth *corev1.Secret + featureGates configv1.FeatureGates wantError string wantCredentialMode lokiv1.CredentialMode } @@ -343,6 +345,45 @@ func TestGCSExtract(t *testing.T) { }, wantCredentialMode: lokiv1.CredentialModeToken, }, + { + name: "invalid for token CCO", + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + TokenCCOAuthEnv: true, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + "key.json": []byte("{\"type\": \"external_account\", \"audience\": \"\", \"service_account_id\": \"\"}"), + }, + }, + wantError: "GCP credentials file contains invalid fields: key.json must not be set for CredentialModeTokenCCO", + }, + { + name: "valid for token CCO", + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + TokenCCOAuthEnv: true, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + }, + }, + tokenAuth: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "token-auth-config"}, + Data: map[string][]byte{ + "service_account.json": []byte("{\"type\": \"external_account\", \"audience\": \"test\", \"service_account_id\": \"\"}"), + }, + }, + wantCredentialMode: lokiv1.CredentialModeTokenCCO, + }, } for _, tst := range table { t.Run(tst.name, func(t *testing.T) { @@ -352,7 +393,7 @@ func TestGCSExtract(t *testing.T) { Type: lokiv1.ObjectStorageSecretGCS, } - opts, err := extractSecrets(spec, tst.secret, nil, configv1.FeatureGates{}) + opts, err := extractSecrets(spec, tst.secret, tst.tokenAuth, tst.featureGates) if tst.wantError == "" { require.NoError(t, err) require.Equal(t, tst.wantCredentialMode, opts.CredentialMode) diff --git a/operator/internal/manifests/openshift/credentialsrequest.go b/operator/internal/manifests/openshift/credentialsrequest.go index 3f202998212a9..7ed2476b3eb40 100644 --- a/operator/internal/manifests/openshift/credentialsrequest.go +++ b/operator/internal/manifests/openshift/credentialsrequest.go @@ -98,6 +98,15 @@ func encodeProviderSpec(env *config.TokenCCOAuthConfig) (*runtime.RawExtension, AzureSubscriptionID: azure.SubscriptionID, AzureTenantID: azure.TenantID, } + case env.GCP != nil: + spec = &cloudcredentialv1.GCPProviderSpec{ + PredefinedRoles: []string{ + "roles/iam.workloadIdentityUser", + "roles/storage.objectAdmin", + }, + Audience: env.GCP.Audience, + ServiceAccountEmail: env.GCP.ServiceAccountEmail, + } } encodedSpec, err := cloudcredentialv1.Codec.EncodeProviderSpec(spec.DeepCopyObject()) diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index ce6fa78273349..18261fec7e95f 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -141,7 +141,9 @@ func ensureObjectStoreCredentials(p *corev1.PodSpec, opts Options) corev1.PodSpe volumes = append(volumes, saTokenVolume(opts)) container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount) - if opts.OpenShift.TokenCCOAuthEnabled() && opts.S3 != nil && opts.S3.STS { + isSTS := opts.S3 != nil && opts.S3.STS + isWIF := opts.GCS != nil && opts.GCS.WorkloadIdentity + if opts.OpenShift.TokenCCOAuthEnabled() && (isSTS || isWIF) { volumes = append(volumes, tokenCCOAuthConfigVolume(opts)) container.VolumeMounts = append(container.VolumeMounts, tokenCCOAuthConfigVolumeMount) } @@ -223,8 +225,14 @@ func tokenAuthCredentials(opts Options) []corev1.EnvVar { envVarFromValue(EnvAzureFederatedTokenFile, ServiceAccountTokenFilePath), } case lokiv1.ObjectStorageSecretGCS: - return []corev1.EnvVar{ - envVarFromValue(EnvGoogleApplicationCredentials, path.Join(secretDirectory, KeyGCPServiceAccountKeyFilename)), + if opts.OpenShift.TokenCCOAuthEnabled() { + return []corev1.EnvVar{ + envVarFromValue(EnvGoogleApplicationCredentials, path.Join(tokenAuthConfigDirectory, KeyGCPManagedServiceAccountKeyFilename)), + } + } else { + return []corev1.EnvVar{ + envVarFromValue(EnvGoogleApplicationCredentials, path.Join(secretDirectory, KeyGCPServiceAccountKeyFilename)), + } } default: return []corev1.EnvVar{} diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 3080f924c11cf..ad930750a7d3d 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -689,6 +689,104 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + desc: "object storage GCS with Workload Identity and OpenShift Managed Credentials", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretGCS, + CredentialMode: lokiv1.CredentialModeTokenCCO, + GCS: &GCSStorageConfig{ + Audience: "test", + WorkloadIdentity: true, + }, + OpenShift: OpenShiftOptions{ + Enabled: true, + CloudCredentials: CloudCredentials{ + SecretName: "cloud-credentials", + SHA1: "deadbeef", + }, + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, + tokenCCOAuthConfigVolumeMount, + }, + Env: []corev1.EnvVar{ + { + Name: EnvGoogleApplicationCredentials, + Value: "/etc/storage/token-auth/service_account.json", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: "test", + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, + { + Name: tokenAuthConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "cloud-credentials", + }, + }, + }, + }, + }, + }, + }, + }, + }, { desc: "object storage S3", opts: Options{ diff --git a/operator/internal/manifests/storage/var.go b/operator/internal/manifests/storage/var.go index ccb69ff27289d..67608c48b7d3e 100644 --- a/operator/internal/manifests/storage/var.go +++ b/operator/internal/manifests/storage/var.go @@ -97,6 +97,8 @@ const ( KeyGCPStorageBucketName = "bucketname" // KeyGCPServiceAccountKeyFilename is the service account key filename containing the Google authentication credentials. KeyGCPServiceAccountKeyFilename = "key.json" + // KeyGCPManagedServiceAccountKeyFilename is the service account key filename for the managed Google service account. + KeyGCPManagedServiceAccountKeyFilename = "service_account.json" // KeySwiftAuthURL is the secret data key for the OpenStack Swift authentication URL. KeySwiftAuthURL = "auth_url"