From 61abe6cac56a6330e520479b8ca3a0835e22caf2 Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 26 Jul 2024 11:50:40 +0300 Subject: [PATCH 1/9] add sse config and build functions Signed-off-by: AvivGuiser --- tempodb/backend/s3/config.go | 39 ++++++++++++++++++++++++++++++++++-- tempodb/backend/s3/s3.go | 34 +++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/tempodb/backend/s3/config.go b/tempodb/backend/s3/config.go index b751b76ca55..8bf9d44f68f 100644 --- a/tempodb/backend/s3/config.go +++ b/tempodb/backend/s3/config.go @@ -1,7 +1,10 @@ package s3 import ( + "errors" "flag" + "fmt" + "strings" "time" "github.com/grafana/dskit/crypto/tls" @@ -10,6 +13,33 @@ import ( "github.com/grafana/tempo/pkg/util" ) +const ( + SignatureVersionV4 = "v4" + SignatureVersionV2 = "v2" + + // SSEKMS config type constant to configure S3 server side encryption using KMS + // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + SSEKMS = "SSE-KMS" + + // SSES3 config type constant to configure S3 server side encryption with AES-256 + // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html + SSES3 = "SSE-S3" +) + +var ( + supportedSignatureVersions = []string{SignatureVersionV4, SignatureVersionV2} + supportedSSETypes = []string{SSEKMS, SSES3} + + errUnsupportedSSEType = errors.New("unsupported S3 SSE type") + errInvalidSSEContext = errors.New("invalid S3 SSE encryption context") +) + +type SSEConfig struct { + Type string `yaml:"type"` + KMSKeyID string `yaml:"kms_key_id"` + KMSEncryptionContext string `yaml:"kms_encryption_context"` +} + type Config struct { tls.ClientConfig `yaml:",inline"` @@ -34,8 +64,9 @@ type Config struct { Metadata map[string]string `yaml:"metadata"` // Deprecated // See https://github.com/grafana/tempo/pull/3006 for more details - NativeAWSAuthEnabled bool `yaml:"native_aws_auth_enabled"` - ListBlocksConcurrency int `yaml:"list_blocks_concurrency"` + NativeAWSAuthEnabled bool `yaml:"native_aws_auth_enabled"` + ListBlocksConcurrency int `yaml:"list_blocks_concurrency"` + SSE SSEConfig `yaml:"sse"` } func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) { @@ -47,6 +78,10 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) f.Var(&cfg.SecretKey, util.PrefixConfig(prefix, "s3.secret_key"), "s3 secret key.") f.Var(&cfg.SessionToken, util.PrefixConfig(prefix, "s3.session_token"), "s3 session token.") f.IntVar(&cfg.ListBlocksConcurrency, util.PrefixConfig(prefix, "s3.list_blocks_concurrency"), 3, "number of concurrent list calls to make to backend") + + f.StringVar(&cfg.SSE.Type, util.PrefixConfig(prefix, "s3.sse.type"), "", fmt.Sprintf("Enable AWS Server Side Encryption. Supported values: %s.", strings.Join(supportedSSETypes, ", "))) + f.StringVar(&cfg.SSE.KMSKeyID, util.PrefixConfig(prefix, "s3.sse.kms-key-id"), "", "KMS Key ID used to encrypt objects in S3") + f.StringVar(&cfg.SSE.KMSEncryptionContext, util.PrefixConfig(prefix, "s3.sse.kms-encryption-context"), "", "KMS Encryption Context used for object encryption. It expects JSON formatted string.") cfg.HedgeRequestsUpTo = 2 } diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index bd19f0ce89b..68bb283ff01 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -3,6 +3,7 @@ package s3 import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -26,6 +27,7 @@ import ( "github.com/go-kit/log/level" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/grafana/tempo/pkg/blockboundary" tempo_io "github.com/grafana/tempo/pkg/io" @@ -693,3 +695,35 @@ func readError(err error) error { } return err } + +func parseKMSEncryptionContext(data string) (map[string]string, error) { + if data == "" { + return nil, nil + } + + decoded := map[string]string{} + err := fmt.Errorf("unable to parse KMS encryption context %w", json.Unmarshal([]byte(data), &decoded)) + return decoded, err +} + +func buildSSEConfig(cfg *Config) (encrypt.ServerSide, error) { + switch cfg.SSE.Type { + case "": + return nil, nil + case SSEKMS: + encryptionCtx, err := parseKMSEncryptionContext(cfg.SSE.KMSEncryptionContext) + if err != nil { + return nil, err + } + + if encryptionCtx == nil { + // To overcome a limitation in Minio which checks interface{} == nil. + return encrypt.NewSSEKMS(cfg.SSE.KMSKeyID, nil) + } + return encrypt.NewSSEKMS(cfg.SSE.KMSKeyID, encryptionCtx) + case SSES3: + return encrypt.NewSSE(), nil + default: + return nil, errUnsupportedSSEType + } +} From f1e9c7a1956e4c6e2a1fdd1023754f1acf063284 Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 26 Jul 2024 17:04:39 +0300 Subject: [PATCH 2/9] add sse to the config and s3 client Signed-off-by: AvivGuiser --- tempodb/backend/s3/s3.go | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index 68bb283ff01..b5495067e6c 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -148,11 +148,33 @@ func internalNew(cfg *Config, confirm bool) (*readerWriter, error) { } func getPutObjectOptions(rw *readerWriter) minio.PutObjectOptions { - return minio.PutObjectOptions{ - PartSize: rw.cfg.PartSize, - UserTags: rw.cfg.Tags, - StorageClass: rw.cfg.StorageClass, - UserMetadata: rw.cfg.Metadata, + sseConfig, err := buildSSEConfig(rw.cfg) + if sseConfig == nil && err == nil { + return minio.PutObjectOptions{ + PartSize: rw.cfg.PartSize, + UserTags: rw.cfg.Tags, + StorageClass: rw.cfg.StorageClass, + UserMetadata: rw.cfg.Metadata, + } + } else { + return minio.PutObjectOptions{ + PartSize: rw.cfg.PartSize, + UserTags: rw.cfg.Tags, + StorageClass: rw.cfg.StorageClass, + UserMetadata: rw.cfg.Metadata, + ServerSideEncryption: sseConfig, + } + } +} + +func getObjectOptions(rw *readerWriter) minio.GetObjectOptions { + sseConfig, err := buildSSEConfig(rw.cfg) + if sseConfig == nil && err == nil { + return minio.GetObjectOptions{} + } else { + return minio.GetObjectOptions{ + ServerSideEncryption: sseConfig, + } } } @@ -542,7 +564,8 @@ func (rw *readerWriter) ReadVersioned(ctx context.Context, name string, keypath } func (rw *readerWriter) readAll(ctx context.Context, name string) ([]byte, error) { - reader, info, _, err := rw.hedgedCore.GetObject(ctx, rw.cfg.Bucket, name, minio.GetObjectOptions{}) + options := getObjectOptions(rw) + reader, info, _, err := rw.hedgedCore.GetObject(ctx, rw.cfg.Bucket, name, options) if err != nil { // do not change or wrap this error // we need to compare the specific err message @@ -554,7 +577,8 @@ func (rw *readerWriter) readAll(ctx context.Context, name string) ([]byte, error } func (rw *readerWriter) readAllWithObjInfo(ctx context.Context, name string) ([]byte, minio.ObjectInfo, error) { - reader, info, _, err := rw.hedgedCore.GetObject(ctx, rw.cfg.Bucket, name, minio.GetObjectOptions{}) + options := getObjectOptions(rw) + reader, info, _, err := rw.hedgedCore.GetObject(ctx, rw.cfg.Bucket, name, options) if err != nil && minio.ToErrorResponse(err).Code == s3.ErrCodeNoSuchKey { return nil, minio.ObjectInfo{}, backend.ErrDoesNotExist } else if err != nil { @@ -570,7 +594,7 @@ func (rw *readerWriter) readAllWithObjInfo(ctx context.Context, name string) ([] } func (rw *readerWriter) readRange(ctx context.Context, objName string, offset int64, buffer []byte) error { - options := minio.GetObjectOptions{} + options := getObjectOptions(rw) err := options.SetRange(offset, offset+int64(len(buffer))) if err != nil { return fmt.Errorf("error setting headers for range read in s3: %w", err) From 188b401fe702e1de95b9a76d5747622380a0ef0f Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 26 Jul 2024 17:11:26 +0300 Subject: [PATCH 3/9] changelog Signed-off-by: AvivGuiser --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf522f3a4f..7370b01db9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ querier: * [CHANGE] fix deprecation warning by switching to DoBatchWithOptions [#4343](https://github.com/grafana/tempo/pull/4343) (@dastrobu) * [CHANGE] **BREAKING CHANGE** The Tempo serverless is now deprecated and will be removed in an upcoming release [#4017](https://github.com/grafana/tempo/pull/4017/) @electron0zero * [CHANGE] tempo-cli: add support for /api/v2/traces endpoint [#4127](https://github.com/grafana/tempo/pull/4127) (@electron0zero) - **BREAKING CHANGE** The `tempo-cli` now uses the `/api/v2/traces` endpoint by default, + **BREAKING CHANGE** The `tempo-cli` now uses the `/api/v2/traces` endpoint by default, please use `--v1` flag to use `/api/traces` endpoint, which was the default in previous versions. * [CHANGE] TraceByID: don't allow concurrent_shards greater than query_shards. [#4074](https://github.com/grafana/tempo/pull/4074) (@electron0zero) * [CHANGE] **BREAKING CHANGE** The dynamic injection of X-Scope-OrgID header for metrics generator remote-writes is changed. If the header is aleady set in per-tenant overrides or global tempo configuration, then it is honored and not overwritten. [#4021](https://github.com/grafana/tempo/pull/4021) (@mdisibio) @@ -181,7 +181,7 @@ querier: * [ENHANCEMENT] Add vParquet4 support to the tempo-cli analyse blocks command [#3868](https://github.com/grafana/tempo/pull/3868) (@stoewer) * [ENHANCEMENT] Improve trace id lookup from Tempo Vulture by selecting a date range [#3874](https://github.com/grafana/tempo/pull/3874) (@javiermolinar) * [ENHANCEMENT] Add native histograms for internal metrics[#3870](https://github.com/grafana/tempo/pull/3870) (@zalegrala) -* [ENHANCEMENT] Expose availability-zone as a cli flag in ingester [#3881](https://github.com/grafana/tempo/pull/3881) +* [ENHANCEMENT] Expose availability-zone as a cli flag in ingester [#3881](https://github.com/grafana/tempo/pull/3881) (@KyriosGN0) * [ENHANCEMENT] Rename batches property of Trace to ResourceSpans to be OTEL compatible [#3895](https://github.com/grafana/tempo/pull/3895) * [ENHANCEMENT] Reduce memory consumption of query-frontend[#3888](https://github.com/grafana/tempo/pull/3888) (@joe-elliott) * [ENHANCEMENT] Reduce log level verbosity for e2e tests[#3900](https://github.com/grafana/tempo/pull/3900) (@javiermolinar) @@ -1297,4 +1297,4 @@ Additionally, default label `span_status` is renamed to `status_code`. * [BUGFIX] S3 multi-part upload errors [#306](https://github.com/grafana/tempo/pull/325) * [BUGFIX] Increase Prometheus `notfound` metric on tempo-vulture. [#301](https://github.com/grafana/tempo/pull/301) * [BUGFIX] Return 404 if searching for a tenant id that does not exist in the backend. [#321](https://github.com/grafana/tempo/pull/321) -* [BUGFIX] Prune in-memory blocks from missing tenants. [#314](https://github.com/grafana/tempo/pull/314) \ No newline at end of file +* [BUGFIX] Prune in-memory blocks from missing tenants. [#314](https://github.com/grafana/tempo/pull/314) From a5fcb22d88453885180a86f2e4d042da7d030212 Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 26 Jul 2024 17:15:58 +0300 Subject: [PATCH 4/9] fix lint Signed-off-by: AvivGuiser --- tempodb/backend/s3/s3.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index b5495067e6c..e780d549317 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -156,14 +156,13 @@ func getPutObjectOptions(rw *readerWriter) minio.PutObjectOptions { StorageClass: rw.cfg.StorageClass, UserMetadata: rw.cfg.Metadata, } - } else { - return minio.PutObjectOptions{ - PartSize: rw.cfg.PartSize, - UserTags: rw.cfg.Tags, - StorageClass: rw.cfg.StorageClass, - UserMetadata: rw.cfg.Metadata, - ServerSideEncryption: sseConfig, - } + } + return minio.PutObjectOptions{ + PartSize: rw.cfg.PartSize, + UserTags: rw.cfg.Tags, + StorageClass: rw.cfg.StorageClass, + UserMetadata: rw.cfg.Metadata, + ServerSideEncryption: sseConfig, } } @@ -171,10 +170,9 @@ func getObjectOptions(rw *readerWriter) minio.GetObjectOptions { sseConfig, err := buildSSEConfig(rw.cfg) if sseConfig == nil && err == nil { return minio.GetObjectOptions{} - } else { - return minio.GetObjectOptions{ - ServerSideEncryption: sseConfig, - } + } + return minio.GetObjectOptions{ + ServerSideEncryption: sseConfig, } } From 64ded3db9d8880b43cc1c5df92aa59c7c1202a08 Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 26 Jul 2024 17:19:41 +0300 Subject: [PATCH 5/9] remove unsued vars Signed-off-by: AvivGuiser --- tempodb/backend/s3/config.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tempodb/backend/s3/config.go b/tempodb/backend/s3/config.go index 8bf9d44f68f..cc249ea7118 100644 --- a/tempodb/backend/s3/config.go +++ b/tempodb/backend/s3/config.go @@ -27,11 +27,9 @@ const ( ) var ( - supportedSignatureVersions = []string{SignatureVersionV4, SignatureVersionV2} - supportedSSETypes = []string{SSEKMS, SSES3} + supportedSSETypes = []string{SSEKMS, SSES3} errUnsupportedSSEType = errors.New("unsupported S3 SSE type") - errInvalidSSEContext = errors.New("invalid S3 SSE encryption context") ) type SSEConfig struct { @@ -83,6 +81,7 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) f.StringVar(&cfg.SSE.KMSKeyID, util.PrefixConfig(prefix, "s3.sse.kms-key-id"), "", "KMS Key ID used to encrypt objects in S3") f.StringVar(&cfg.SSE.KMSEncryptionContext, util.PrefixConfig(prefix, "s3.sse.kms-encryption-context"), "", "KMS Encryption Context used for object encryption. It expects JSON formatted string.") cfg.HedgeRequestsUpTo = 2 + } func (cfg *Config) PathMatches(other *Config) bool { From de6f00c1d27590317194dfaa7333382f49d479a7 Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 26 Jul 2024 17:25:37 +0300 Subject: [PATCH 6/9] fmt Signed-off-by: AvivGuiser --- tempodb/backend/s3/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tempodb/backend/s3/config.go b/tempodb/backend/s3/config.go index cc249ea7118..cd4e16ca6eb 100644 --- a/tempodb/backend/s3/config.go +++ b/tempodb/backend/s3/config.go @@ -81,7 +81,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) f.StringVar(&cfg.SSE.KMSKeyID, util.PrefixConfig(prefix, "s3.sse.kms-key-id"), "", "KMS Key ID used to encrypt objects in S3") f.StringVar(&cfg.SSE.KMSEncryptionContext, util.PrefixConfig(prefix, "s3.sse.kms-encryption-context"), "", "KMS Encryption Context used for object encryption. It expects JSON formatted string.") cfg.HedgeRequestsUpTo = 2 - } func (cfg *Config) PathMatches(other *Config) bool { From 0fc6eabe56a85f0917ad6dce2e502374240ac66b Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Sat, 3 Aug 2024 19:33:04 +0300 Subject: [PATCH 7/9] added check for missing kms key id Signed-off-by: AvivGuiser --- tempodb/backend/s3/s3.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index e780d549317..17491c1be87 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -144,6 +144,7 @@ func internalNew(cfg *Config, confirm bool) (*readerWriter, error) { core: core, hedgedCore: hedgedCore, } + return rw, nil } @@ -724,7 +725,7 @@ func parseKMSEncryptionContext(data string) (map[string]string, error) { } decoded := map[string]string{} - err := fmt.Errorf("unable to parse KMS encryption context %w", json.Unmarshal([]byte(data), &decoded)) + err := json.Unmarshal([]byte(data), &decoded) return decoded, err } @@ -733,16 +734,20 @@ func buildSSEConfig(cfg *Config) (encrypt.ServerSide, error) { case "": return nil, nil case SSEKMS: - encryptionCtx, err := parseKMSEncryptionContext(cfg.SSE.KMSEncryptionContext) - if err != nil { - return nil, err - } + if cfg.SSE.KMSKeyID == "" { + return nil, errors.New("KMSKeyID is missing") + } else { + encryptionCtx, err := parseKMSEncryptionContext(cfg.SSE.KMSEncryptionContext) + if err != nil { + return nil, err + } + if encryptionCtx == nil { + // To overcome a limitation in Minio which checks interface{} == nil. - if encryptionCtx == nil { - // To overcome a limitation in Minio which checks interface{} == nil. - return encrypt.NewSSEKMS(cfg.SSE.KMSKeyID, nil) + return encrypt.NewSSEKMS(cfg.SSE.KMSKeyID, nil) + } + return encrypt.NewSSEKMS(cfg.SSE.KMSKeyID, encryptionCtx) } - return encrypt.NewSSEKMS(cfg.SSE.KMSKeyID, encryptionCtx) case SSES3: return encrypt.NewSSE(), nil default: From 78f4ba337d2fcf5fce7a4d4a8902ffb4e1a7059e Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 9 Aug 2024 18:53:53 +0300 Subject: [PATCH 8/9] create sse config when creating a minio client Signed-off-by: AvivGuiser --- tempodb/backend/s3/s3.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index 17491c1be87..5d78847c821 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -41,6 +41,7 @@ type readerWriter struct { cfg *Config core *minio.Core hedgedCore *minio.Core + sse encrypt.ServerSide } var tracer = otel.Tracer("tempodb/backend/s3") @@ -138,42 +139,35 @@ func internalNew(cfg *Config, confirm bool) (*readerWriter, error) { } } + encryption, err := buildSSEConfig(cfg) + if err != nil { + return nil, fmt.Errorf("returned Error when trying to configure Server Side Encryption: %w", err) + } + rw := &readerWriter{ logger: l, cfg: cfg, core: core, hedgedCore: hedgedCore, + sse: encryption, } return rw, nil } func getPutObjectOptions(rw *readerWriter) minio.PutObjectOptions { - sseConfig, err := buildSSEConfig(rw.cfg) - if sseConfig == nil && err == nil { - return minio.PutObjectOptions{ - PartSize: rw.cfg.PartSize, - UserTags: rw.cfg.Tags, - StorageClass: rw.cfg.StorageClass, - UserMetadata: rw.cfg.Metadata, - } - } return minio.PutObjectOptions{ PartSize: rw.cfg.PartSize, UserTags: rw.cfg.Tags, StorageClass: rw.cfg.StorageClass, UserMetadata: rw.cfg.Metadata, - ServerSideEncryption: sseConfig, + ServerSideEncryption: rw.sse, } } func getObjectOptions(rw *readerWriter) minio.GetObjectOptions { - sseConfig, err := buildSSEConfig(rw.cfg) - if sseConfig == nil && err == nil { - return minio.GetObjectOptions{} - } return minio.GetObjectOptions{ - ServerSideEncryption: sseConfig, + ServerSideEncryption: rw.sse, } } From 1b55e2da99e8bab8356950523803fb44785c6815 Mon Sep 17 00:00:00 2001 From: AvivGuiser Date: Fri, 1 Nov 2024 23:09:16 +0200 Subject: [PATCH 9/9] add changelog and docs Signed-off-by: AvivGuiser --- docs/sources/tempo/configuration/_index.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 37c404360e4..d90a99bc6c1 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -679,6 +679,7 @@ query_frontend: # NOTE: Requires `duration_slo` AND `throughput_bytes_slo` to be configured. [duration_slo: | default = 0s ] + # If set to a non-zero value, it's value will be used to decide if metadata query is within SLO or not. # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data. [throughput_bytes_slo: | default = 0 ] @@ -1111,6 +1112,22 @@ storage: # See the [S3 documentation on object tagging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) for more detail. [tags: ] + + [sse: ]: + # Optional + # Example: type: SSE-S3 + # Type of encryption to use with s3 bucket, either SSE-KMS or SSE-S3 + [type: string]: + + # Optional + # Example: kms_key_id: "1234abcd-12ab-34cd-56ef-1234567890ab" + # the kms key id is the identification of the key in an account or region + kms_key_id: + # Optional + # Example: kms_encryption_context: "encryptionContext": {"department": "10103.0"} + # KMS Encryption Context used for object encryption. It expects JSON formatted string + kms_encryption_context: + # azure configuration. Will be used only if value of backend is "azure" # EXPERIMENTAL azure: