operator: Fix storing authentication credentials in the Loki ConfigMap (

grafana#11357) Co-authored-by: Robert Jacob <[email protected]> Co-authored-by: Robert Jacob <[email protected]>
JStickler · Dec 11, 2023 · c573def · c573def
1 parent 8dde7b9
commit c573def
Show file tree

Hide file tree

Showing 30 changed files with 1,728 additions and 388 deletions.
diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -1,5 +1,6 @@
 ## Main
 
+- [11357](https://github.com/grafana/loki/pull/11357) **periklis**: Fix storing authentication credentials in the Loki ConfigMap
 - [11393](https://github.com/grafana/loki/pull/11393) **periklis**: Add infra annotations for OpenShift based deployments
 - [11094](https://github.com/grafana/loki/pull/11094) **periklis**: Add support for blocking queries per tenant
 - [11288](https://github.com/grafana/loki/pull/11288) **periklis**: Fix custom CA for object-store in ruler component

diff --git a/operator/cmd/loki-broker/main.go b/operator/cmd/loki-broker/main.go
@@ -57,8 +57,6 @@ func (c *config) registerFlags(f *flag.FlagSet) {
 	f.StringVar(&c.objectStorage.S3.Endpoint, "object-storage.s3.endpoint", "", "The S3 endpoint location.")
 	f.StringVar(&c.objectStorage.S3.Buckets, "object-storage.s3.buckets", "", "A comma-separated list of S3 buckets.")
 	f.StringVar(&c.objectStorage.S3.Region, "object-storage.s3.region", "", "An S3 region.")
-	f.StringVar(&c.objectStorage.S3.AccessKeyID, "object-storage.s3.access-key-id", "", "The access key id for S3.")
-	f.StringVar(&c.objectStorage.S3.AccessKeySecret, "object-storage.s3.access-key-secret", "", "The access key secret for S3.")
 	// Input and output file/dir options
 	f.StringVar(&c.crFilepath, "custom-resource.path", "", "Path to a custom resource YAML file.")
 	f.StringVar(&c.writeToDir, "output.write-dir", "", "write each file to the specified directory.")
@@ -88,14 +86,6 @@ func (c *config) validateFlags(log logr.Logger) {
 		log.Info("-object-storage.s3.buckets flag is required")
 		os.Exit(1)
 	}
-	if cfg.objectStorage.S3.AccessKeyID == "" {
-		log.Info("-object-storage.s3.access.key.id flag is required")
-		os.Exit(1)
-	}
-	if cfg.objectStorage.S3.AccessKeySecret == "" {
-		log.Info("-object-storage.s3.access.key.secret flag is required")
-		os.Exit(1)
-	}
 	// Validate feature flags
 	if cfg.featureFlags.LokiStackAlerts && !cfg.featureFlags.ServiceMonitors {
 		log.Info("-with-prometheus-alerts flag requires -with-service-monitors")

diff --git a/operator/docs/lokistack/object_storage.md b/operator/docs/lokistack/object_storage.md
@@ -51,7 +51,7 @@ _Note_: Upon setting up LokiStack for any object storage provider, you should co
       --from-literal=sse_kms_encryption_context="<OPTIONAL_AWS_SSE_KMS_ENCRYPTION_CONTEXT_JSON>"
     ```
     
-    See also official docs on [AWS KMS Key ID](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id) and [AWS KMS Encryption Context](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context).
+    See also official docs on [AWS KMS Key ID](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id) and [AWS KMS Encryption Context](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) (**Note:** Only content without newlines allowed, because it is exposed via environment variable to the containers).
 
     or with `SSE-S3` encryption
 

diff --git a/operator/internal/handlers/internal/storage/secrets.go b/operator/internal/handlers/internal/storage/secrets.go
@@ -1,18 +1,29 @@
 package storage
 
 import (
+	"crypto/sha1"
+	"fmt"
+	"sort"
+
 	"github.com/ViaQ/logerr/v2/kverrors"
 	corev1 "k8s.io/api/core/v1"
 
 	lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
 	"github.com/grafana/loki/operator/internal/manifests/storage"
 )
 
+var hashSeparator = []byte(",")
+
 // ExtractSecret reads a k8s secret into a manifest object storage struct if valid.
 func ExtractSecret(s *corev1.Secret, secretType lokiv1.ObjectStorageSecretType) (*storage.Options, error) {
-	var err error
+	hash, err := hashSecretData(s)
+	if err != nil {
+		return nil, kverrors.Wrap(err, "error calculating hash for secret", "type", secretType)
+	}
+
 	storageOpts := storage.Options{
 		SecretName:  s.Name,
+		SecretSHA1:  hash,
 		SharedStore: secretType,
 	}
 
@@ -37,48 +48,75 @@ func ExtractSecret(s *corev1.Secret, secretType lokiv1.ObjectStorageSecretType)
 	return &storageOpts, nil
 }
 
+func hashSecretData(s *corev1.Secret) (string, error) {
+	keys := make([]string, 0, len(s.Data))
+	for k := range s.Data {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	h := sha1.New()
+	for _, k := range keys {
+		if _, err := h.Write([]byte(k)); err != nil {
+			return "", err
+		}
+
+		if _, err := h.Write(hashSeparator); err != nil {
+			return "", err
+		}
+
+		if _, err := h.Write(s.Data[k]); err != nil {
+			return "", err
+		}
+
+		if _, err := h.Write(hashSeparator); err != nil {
+			return "", err
+		}
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
 func extractAzureConfigSecret(s *corev1.Secret) (*storage.AzureStorageConfig, error) {
 	// Extract and validate mandatory fields
-	env := s.Data["environment"]
+	env := s.Data[storage.KeyAzureEnvironmentName]
 	if len(env) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "environment")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAzureEnvironmentName)
 	}
-	container := s.Data["container"]
+	container := s.Data[storage.KeyAzureStorageContainerName]
 	if len(container) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "container")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAzureStorageContainerName)
 	}
-	name := s.Data["account_name"]
+	name := s.Data[storage.KeyAzureStorageAccountName]
 	if len(name) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "account_name")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAzureStorageAccountName)
 	}
-	key := s.Data["account_key"]
+	key := s.Data[storage.KeyAzureStorageAccountKey]
 	if len(key) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "account_key")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAzureStorageAccountKey)
 	}
 
 	// Extract and validate optional fields
-	endpointSuffix := s.Data["endpoint_suffix"]
+	endpointSuffix := s.Data[storage.KeyAzureStorageEndpointSuffix]
 
 	return &storage.AzureStorageConfig{
 		Env:            string(env),
 		Container:      string(container),
-		AccountName:    string(name),
-		AccountKey:     string(key),
 		EndpointSuffix: string(endpointSuffix),
 	}, nil
 }
 
 func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error) {
 	// Extract and validate mandatory fields
-	bucket := s.Data["bucketname"]
+	bucket := s.Data[storage.KeyGCPStorageBucketName]
 	if len(bucket) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "bucketname")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyGCPStorageBucketName)
 	}
 
 	// Check if google authentication credentials is provided
-	keyJSON := s.Data["key.json"]
+	keyJSON := s.Data[storage.KeyGCPServiceAccountKeyFilename]
 	if len(keyJSON) == 0 {
-		return nil, kverrors.New("missing google authentication credentials", "field", "key.json")
+		return nil, kverrors.New("missing google authentication credentials", "field", storage.KeyGCPServiceAccountKeyFilename)
 	}
 
 	return &storage.GCSStorageConfig{
@@ -88,38 +126,36 @@ func extractGCSConfigSecret(s *corev1.Secret) (*storage.GCSStorageConfig, error)
 
 func extractS3ConfigSecret(s *corev1.Secret) (*storage.S3StorageConfig, error) {
 	// Extract and validate mandatory fields
-	endpoint := s.Data["endpoint"]
+	endpoint := s.Data[storage.KeyAWSEndpoint]
 	if len(endpoint) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "endpoint")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAWSEndpoint)
 	}
-	buckets := s.Data["bucketnames"]
+	buckets := s.Data[storage.KeyAWSBucketNames]
 	if len(buckets) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "bucketnames")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAWSBucketNames)
 	}
-	id := s.Data["access_key_id"]
+	id := s.Data[storage.KeyAWSAccessKeyID]
 	if len(id) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "access_key_id")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAWSAccessKeyID)
 	}
-	secret := s.Data["access_key_secret"]
+	secret := s.Data[storage.KeyAWSAccessKeySecret]
 	if len(secret) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "access_key_secret")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAWSAccessKeySecret)
 	}
 
 	// Extract and validate optional fields
-	region := s.Data["region"]
+	region := s.Data[storage.KeyAWSRegion]
 
 	sseCfg, err := extractS3SSEConfig(s.Data)
 	if err != nil {
 		return nil, err
 	}
 
 	return &storage.S3StorageConfig{
-		Endpoint:        string(endpoint),
-		Buckets:         string(buckets),
-		AccessKeyID:     string(id),
-		AccessKeySecret: string(secret),
-		Region:          string(region),
-		SSE:             sseCfg,
+		Endpoint: string(endpoint),
+		Buckets:  string(buckets),
+		Region:   string(region),
+		SSE:      sseCfg,
 	}, nil
 }
 
@@ -129,20 +165,20 @@ func extractS3SSEConfig(d map[string][]byte) (storage.S3SSEConfig, error) {
 		kmsKeyId, kmsEncryptionCtx string
 	)
 
-	switch sseType = storage.S3SSEType(d["sse_type"]); sseType {
+	switch sseType = storage.S3SSEType(d[storage.KeyAWSSSEType]); sseType {
 	case storage.SSEKMSType:
-		kmsEncryptionCtx = string(d["sse_kms_encryption_context"])
-		kmsKeyId = string(d["sse_kms_key_id"])
+		kmsEncryptionCtx = string(d[storage.KeyAWSSseKmsEncryptionContext])
+		kmsKeyId = string(d[storage.KeyAWSSseKmsKeyID])
 		if kmsKeyId == "" {
-			return storage.S3SSEConfig{}, kverrors.New("missing secret field", "field", "sse_kms_key_id")
+			return storage.S3SSEConfig{}, kverrors.New("missing secret field", "field", storage.KeyAWSSseKmsKeyID)
 		}
 
 	case storage.SSES3Type:
 	case "":
 		return storage.S3SSEConfig{}, nil
 
 	default:
-		return storage.S3SSEConfig{}, kverrors.New("unsupported secret field value (Supported: SSE-KMS, SSE-S3)", "field", "sse_type", "value", sseType)
+		return storage.S3SSEConfig{}, kverrors.New("unsupported secret field value (Supported: SSE-KMS, SSE-S3)", "field", storage.KeyAWSSSEType, "value", sseType)
 	}
 
 	return storage.S3SSEConfig{
@@ -154,57 +190,55 @@ func extractS3SSEConfig(d map[string][]byte) (storage.S3SSEConfig, error) {
 
 func extractSwiftConfigSecret(s *corev1.Secret) (*storage.SwiftStorageConfig, error) {
 	// Extract and validate mandatory fields
-	url := s.Data["auth_url"]
+	url := s.Data[storage.KeySwiftAuthURL]
 	if len(url) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "auth_url")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftAuthURL)
 	}
-	username := s.Data["username"]
+	username := s.Data[storage.KeySwiftUsername]
 	if len(username) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "username")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftUsername)
 	}
-	userDomainName := s.Data["user_domain_name"]
+	userDomainName := s.Data[storage.KeySwiftUserDomainName]
 	if len(userDomainName) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "user_domain_name")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftUserDomainName)
 	}
-	userDomainID := s.Data["user_domain_id"]
+	userDomainID := s.Data[storage.KeySwiftUserDomainID]
 	if len(userDomainID) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "user_domain_id")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftUserDomainID)
 	}
-	userID := s.Data["user_id"]
+	userID := s.Data[storage.KeySwiftUserID]
 	if len(userID) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "user_id")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftUserID)
 	}
-	password := s.Data["password"]
+	password := s.Data[storage.KeySwiftPassword]
 	if len(password) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "password")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftPassword)
 	}
-	domainID := s.Data["domain_id"]
+	domainID := s.Data[storage.KeySwiftDomainID]
 	if len(domainID) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "domain_id")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftDomainID)
 	}
-	domainName := s.Data["domain_name"]
+	domainName := s.Data[storage.KeySwiftDomainName]
 	if len(domainName) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "domain_name")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftDomainName)
 	}
-	containerName := s.Data["container_name"]
+	containerName := s.Data[storage.KeySwiftContainerName]
 	if len(containerName) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "container_name")
+		return nil, kverrors.New("missing secret field", "field", storage.KeySwiftContainerName)
 	}
 
 	// Extract and validate optional fields
-	projectID := s.Data["project_id"]
-	projectName := s.Data["project_name"]
-	projectDomainID := s.Data["project_domain_id"]
-	projectDomainName := s.Data["project_domain_name"]
-	region := s.Data["region"]
+	projectID := s.Data[storage.KeySwiftProjectID]
+	projectName := s.Data[storage.KeySwiftProjectName]
+	projectDomainID := s.Data[storage.KeySwiftProjectDomainId]
+	projectDomainName := s.Data[storage.KeySwiftProjectDomainName]
+	region := s.Data[storage.KeySwiftRegion]
 
 	return &storage.SwiftStorageConfig{
 		AuthURL:           string(url),
-		Username:          string(username),
 		UserDomainName:    string(userDomainName),
 		UserDomainID:      string(userDomainID),
 		UserID:            string(userID),
-		Password:          string(password),
 		DomainID:          string(domainID),
 		DomainName:        string(domainName),
 		ProjectID:         string(projectID),
@@ -218,28 +252,25 @@ func extractSwiftConfigSecret(s *corev1.Secret) (*storage.SwiftStorageConfig, er
 
 func extractAlibabaCloudConfigSecret(s *corev1.Secret) (*storage.AlibabaCloudStorageConfig, error) {
 	// Extract and validate mandatory fields
-	endpoint := s.Data["endpoint"]
+	endpoint := s.Data[storage.KeyAlibabaCloudEndpoint]
 	if len(endpoint) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "endpoint")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAlibabaCloudEndpoint)
 	}
-	bucket := s.Data["bucket"]
+	bucket := s.Data[storage.KeyAlibabaCloudBucket]
 	if len(bucket) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "bucket")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAlibabaCloudBucket)
 	}
-	// TODO buckets are comma-separated list
-	id := s.Data["access_key_id"]
+	id := s.Data[storage.KeyAlibabaCloudAccessKeyID]
 	if len(id) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "access_key_id")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAlibabaCloudAccessKeyID)
 	}
-	secret := s.Data["secret_access_key"]
+	secret := s.Data[storage.KeyAlibabaCloudSecretAccessKey]
 	if len(secret) == 0 {
-		return nil, kverrors.New("missing secret field", "field", "secret_access_key")
+		return nil, kverrors.New("missing secret field", "field", storage.KeyAlibabaCloudSecretAccessKey)
 	}
 
 	return &storage.AlibabaCloudStorageConfig{
-		Endpoint:        string(endpoint),
-		Bucket:          string(bucket),
-		AccessKeyID:     string(id),
-		SecretAccessKey: string(secret),
+		Endpoint: string(endpoint),
+		Bucket:   string(bucket),
 	}, nil
 }