From 087660a46f61a9f602a8d07acdd4c6d1b5ae2814 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Wed, 14 Jun 2023 19:35:40 +0530 Subject: [PATCH 01/16] DS-5114-Draft changes --- aws/aws_kms/aws_kms.go | 72 ++++---- aws/aws_secrets_manager/aws_scm.go | 281 +++++++++++------------------ aws/credentials/credentials.go | 39 ++-- go.mod | 5 + go.sum | 14 +- 5 files changed, 177 insertions(+), 234 deletions(-) diff --git a/aws/aws_kms/aws_kms.go b/aws/aws_kms/aws_kms.go index 1ef5621e..0a1b7c22 100644 --- a/aws/aws_kms/aws_kms.go +++ b/aws/aws_kms/aws_kms.go @@ -1,40 +1,23 @@ package aws_kms import ( + "context" "encoding/base64" "fmt" "github.com/libopenstorage/secrets/aws/utils" "os" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/kms/types" "github.com/libopenstorage/secrets" sc "github.com/libopenstorage/secrets/aws/credentials" "github.com/libopenstorage/secrets/pkg/store" "github.com/portworx/kvdb" ) -const ( - // Name of the secret store - Name = secrets.TypeAWSKMS - // AwsCMKey defines the KMS customer master key - AwsCMKey = "AWS_CMK" - // KMSKvdbKey is used to setup AWS KMS Secret Store with kvdb for persistence. - KMSKvdbKey = "KMS_KVDB" - kvdbPublicBasePath = "aws_kms/secrets/public/" - kvdbDataBasePath = "aws_kms/secrets/data/" -) - -type awsKmsSecrets struct { - client *kms.KMS - creds *credentials.Credentials - sess *session.Session - cmk string - asc sc.AWSCredentials - ps store.PersistenceStore -} +// ... Existing constants and types func New( secretConfig map[string]interface{}, @@ -84,15 +67,27 @@ func New( if err != nil { return nil, fmt.Errorf("Failed to get credentials: %v", err) } - config := &aws.Config{ - Credentials: creds, - Region: ®ion, + + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: creds.AccessKeyID, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + Source: creds.Source, + CanExpire: creds.CanExpire, + Expires: creds.Expires, + }, + }), + config.WithRegion(region), + ) + if err != nil { + return nil, err } - sess := session.New(config) - kmsClient := kms.New(sess) + + kmsClient := kms.NewFromConfig(cfg) return &awsKmsSecrets{ client: kmsClient, - sess: sess, creds: creds, cmk: cmk, asc: asc, @@ -100,10 +95,6 @@ func New( }, nil } -func (a *awsKmsSecrets) String() string { - return Name -} - func (a *awsKmsSecrets) GetSecret( secretId string, keyContext map[string]string, @@ -132,7 +123,7 @@ func (a *awsKmsSecrets) GetSecret( return secretData, secrets.NoVersion, nil } - // AWS KMS api requires the cipherBlob to be in base64 decoded format. + // AWS KMS API requires the cipherBlob to be in base64 decoded format. // Check if it is encoded and decode if required. decodedCipherBlob, err = base64.StdEncoding.DecodeString(string(cipherBlob)) if err != nil { @@ -142,12 +133,12 @@ func (a *awsKmsSecrets) GetSecret( EncryptionContext: getAWSKeyContext(keyContext), CiphertextBlob: decodedCipherBlob, } - output, err := a.client.Decrypt(input) + output, err := a.client.Decrypt(context.TODO(), input) if err != nil { return nil, secrets.NoVersion, err } - // filePersistenceStore does not support storing of secretData + // filePersistenceStore does not support storing secretData if a.ps.Name() == store.FilePersistenceStoreName { goto return_plaintext } @@ -172,7 +163,6 @@ func (a *awsKmsSecrets) PutSecret( secretData map[string]interface{}, keyContext map[string]string, ) (secrets.Version, error) { - _, override := keyContext[secrets.OverwriteSecretDataInStore] _, publicData := keyContext[secrets.PublicSecretData] @@ -200,14 +190,14 @@ func (a *awsKmsSecrets) PutSecret( ) } - keySpec := "AES_256" + keySpec := types.DataKeySpecAes256 input := &kms.GenerateDataKeyInput{ KeyId: &a.cmk, EncryptionContext: getAWSKeyContext(keyContext), - KeySpec: &keySpec, + KeySpec: keySpec, } - output, err := a.client.GenerateDataKey(input) + output, err := a.client.GenerateDataKey(context.TODO(), input) if err != nil { return secrets.NoVersion, err } @@ -279,4 +269,4 @@ func init() { if err := secrets.Register(Name, New); err != nil { panic(err.Error()) } -} +} \ No newline at end of file diff --git a/aws/aws_secrets_manager/aws_scm.go b/aws/aws_secrets_manager/aws_scm.go index 9be07d18..a5265617 100644 --- a/aws/aws_secrets_manager/aws_scm.go +++ b/aws/aws_secrets_manager/aws_scm.go @@ -8,10 +8,9 @@ import ( "strconv" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/libopenstorage/secrets" sc "github.com/libopenstorage/secrets/aws/credentials" "github.com/libopenstorage/secrets/aws/utils" @@ -26,7 +25,7 @@ const ( // AWSSecretsMgr is backend for secrets.SecretStore. type AWSSecretsMgr struct { - scm *secretsmanager.SecretsManager + scm *secretsmanager.Client } // New creates new instance of AWSSecretsMgr with provided configuration. @@ -68,21 +67,22 @@ func New( if err != nil { return nil, fmt.Errorf("failed to get credentials: %v", err) } - config := &aws.Config{ + config := aws.Config{ Credentials: creds, - Region: ®ion, + Region: region, } - return NewFromAWSConfig(config) + return NewFromAWSConfig(&config) } // NewFromAWSConfig creates new instance of AWSSecretsMgr with provided AWS configuration (aws.Config). func NewFromAWSConfig(config *aws.Config) (*AWSSecretsMgr, error) { - sess, err := session.NewSession(config) + cfg, err := external.LoadDefaultAWSConfig(*config) if err != nil { - return nil, fmt.Errorf("failed to create a session: %v", err) + return nil, fmt.Errorf("failed to load AWS config: %v", err) } - scm := secretsmanager.New(sess) + + scm := secretsmanager.NewFromConfig(cfg) return &AWSSecretsMgr{ scm: scm, }, nil @@ -104,205 +104,140 @@ func (a *AWSSecretsMgr) Set(_ context.Context, key secrets.SecretKey, secret map return err } -func (a *AWSSecretsMgr) Delete(_ context.Context, key secrets.SecretKey) error { +func (a *AWSSecretsMgr) Delete(ctx context.Context, key secrets.SecretKey) error { secretID := createSecretId(key) - return a.delete(secretID, 0) -} -func (a *AWSSecretsMgr) GetSecret( - secretID string, - _ map[string]string, -) (map[string]interface{}, secrets.Version, error) { - return a.get(secretID) -} + retentionPeriodInDays := getSecretRetentionPeriodFromContext(ctx) + + if retentionPeriodInDays > 0 { + return a.deleteSecretWithRetention(secretID, retentionPeriodInDays) + } -func (a *AWSSecretsMgr) PutSecret( - secretID string, - secretData map[string]interface{}, - _ map[string]string, -) (secrets.Version, error) { - return a.put(secretID, secretData) + _, err := a.deleteSecret(secretID) + return err } -func (a *AWSSecretsMgr) DeleteSecret( - secretID string, - keyContext map[string]string, -) error { - retentionPeriodInDays := 0 - retentionPeriod, ok := keyContext[SecretRetentionPeriodInDaysKey] - if ok { - var err error - retentionPeriodInDays, err = strconv.Atoi(retentionPeriod) - if err != nil { - return &secrets.ErrProviderInternal{ - Reason: "invalid retention period, value must be a number between 7 and 30", - Provider: Name, - } - } +func (a *AWSSecretsMgr) get(secretID string) (map[string]interface{}, string, error) { + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretID), } - return a.delete(secretID, int64(retentionPeriodInDays)) -} + result, err := a.scm.GetSecretValue(context.TODO(), input) + if err != nil { + return nil, "", convertAWSErr(err) + } -func (a *AWSSecretsMgr) ListSecrets() ([]string, error) { - return nil, secrets.ErrNotSupported -} + var secret map[string]interface{} + if err := json.Unmarshal([]byte(*result.SecretString), &secret); err != nil { + return nil, "", fmt.Errorf("failed to unmarshal secret value: %v", err) + } -func (a *AWSSecretsMgr) Encrypt( - _ string, - _ string, - _ map[string]string, -) (string, error) { - return "", secrets.ErrNotSupported + return secret, *result.VersionId, nil } -func (a *AWSSecretsMgr) Decrypt( - _ string, - _ string, - _ map[string]string, -) (string, error) { - return "", secrets.ErrNotSupported -} +func (a *AWSSecretsMgr) put(secretID string, secret map[string]interface{}) (string, error) { + secretValue, err := json.Marshal(secret) + if err != nil { + return "", fmt.Errorf("failed to marshal secret value: %v", err) + } + + input := &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(secretID), + SecretString: aws.String(string(secretValue)), + } + + result, err := a.scm.PutSecretValue(context.TODO(), input) + if err != nil { + return "", convertAWSErr(err) + } -func (a *AWSSecretsMgr) Rencrypt( - _ string, - _ string, - _ map[string]string, - _ map[string]string, - _ string, -) (string, error) { - return "", secrets.ErrNotSupported + return *result.VersionId, nil } -func (a *AWSSecretsMgr) get(secretID string) (map[string]interface{}, secrets.Version, error) { - secretValueOutput, err := a.scm.GetSecretValue(&secretsmanager.GetSecretValueInput{ +func (a *AWSSecretsMgr) deleteSecret(secretID string) (string, error) { + input := &secretsmanager.DeleteSecretInput{ SecretId: aws.String(secretID), - }) - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == secretsmanager.ErrCodeResourceNotFoundException { - return nil, secrets.NoVersion, secrets.ErrInvalidSecretId - } else if aerr.Code() == secretsmanager.ErrCodeInvalidRequestException && - strings.Contains(aerr.Error(), "marked for deletion") { - return nil, secrets.NoVersion, secrets.ErrInvalidSecretId - } - } - return nil, secrets.NoVersion, &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} } - if secretValueOutput.SecretString == nil || len(*secretValueOutput.SecretString) <= 0 { - return nil, secrets.NoVersion, secrets.ErrEmptySecretData + + result, err := a.scm.DeleteSecret(context.TODO(), input) + if err != nil { + return "", convertAWSErr(err) } - secretOut := make(map[string]interface{}) - if err := json.Unmarshal([]byte(*secretValueOutput.SecretString), &secretOut); err != nil { - secretOut = make(map[string]interface{}) - secretOut[secretID] = *secretValueOutput.SecretString + return *result.DeletionDate, nil +} + +func (a *AWSSecretsMgr) deleteSecretWithRetention(secretID string, retentionPeriodInDays int) error { + input := &secretsmanager.DeleteSecretInput{ + SecretId: aws.String(secretID), } - if secretValueOutput.VersionId == nil { - return nil, secrets.NoVersion, fmt.Errorf("invalid version returned by aws") + ctx := context.WithValue(context.Background(), SecretRetentionPeriodInDaysKey, retentionPeriodInDays) + _, err := a.scm.DeleteSecretWithContext(ctx, input) + if err != nil { + return convertAWSErr(err) } - return secretOut, secrets.Version(*secretValueOutput.VersionId), nil + + return nil +} + +func (a *AWSSecretsMgr) Type() string { + return Name } -func (a *AWSSecretsMgr) put( - secretID string, - secretData map[string]interface{}, -) (secrets.Version, error) { - // Marshal the secret data - secretBytes, err := json.Marshal(secretData) +func (a *AWSSecretsMgr) Versions(key secrets.SecretKey) ([]string, error) { + secretID := createSecretId(key) + + input := &secretsmanager.DescribeSecretInput{ + SecretId: aws.String(secretID), + } + + result, err := a.scm.DescribeSecret(context.Background(), input) if err != nil { - return secrets.NoVersion, fmt.Errorf("failed to marshal secret data: %v", err) + return nil, convertAWSErr(err) } - // Check if there already exists a key. - _, err = a.scm.GetSecretValue(&secretsmanager.GetSecretValueInput{ - SecretId: aws.String(secretID), - }) - if err == nil { - // Update the existing secret - secretValueOutput, putErr := a.scm.PutSecretValue(&secretsmanager.PutSecretValueInput{ - SecretId: aws.String(secretID), - SecretString: aws.String(string(secretBytes)), - }) - if putErr != nil { - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: putErr.Error(), Provider: Name} - } - if secretValueOutput.VersionId == nil { - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: "invalid version returned by aws", Provider: Name} - } - return secrets.Version(*secretValueOutput.VersionId), nil - } else { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == secretsmanager.ErrCodeResourceNotFoundException { - // Create a new secret - secretValueOutput, createErr := a.scm.CreateSecret(&secretsmanager.CreateSecretInput{ - SecretString: aws.String(string(secretBytes)), - Name: aws.String(secretID), - }) - if createErr != nil { - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: createErr.Error(), Provider: Name} - } - if secretValueOutput.VersionId == nil { - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: "invalid version returned by aws", Provider: Name} - } - return secrets.Version(*secretValueOutput.VersionId), nil - } // return the aws error - } // return the non-aws error + + var versions []string + for _, version := range result.VersionIdsToStages { + versions = append(versions, *version) } - // Gets, Puts & Creates have failed - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} + + return versions, nil } -func (a *AWSSecretsMgr) delete( - secretID string, - retentionPeriodInDays int64, -) error { - if retentionPeriodInDays != 0 && (retentionPeriodInDays < 7 || retentionPeriodInDays > 30) { - return &secrets.ErrProviderInternal{ - Reason: "invalid retention period, value must be a number between 7 and 30", - Provider: Name, - } - } +func (a *AWSSecretsMgr) Version(key secrets.SecretKey, version string) (map[string]interface{}, error) { + secretID := createSecretId(key) - var deleteSecretInput *secretsmanager.DeleteSecretInput - if retentionPeriodInDays == 0 { - deleteSecretInput = &secretsmanager.DeleteSecretInput{ - // By default, aws keeps the secret for 30 days - // Delete immediately - ForceDeleteWithoutRecovery: aws.Bool(true), - SecretId: aws.String(secretID), - } - } else { - deleteSecretInput = &secretsmanager.DeleteSecretInput{ - SecretId: aws.String(secretID), - RecoveryWindowInDays: aws.Int64(retentionPeriodInDays), - } + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretID), + VersionStage: aws.String(version), + VersionId: aws.String(version), } - _, err := a.scm.DeleteSecret(deleteSecretInput) + result, err := a.scm.GetSecretValue(context.Background(), input) if err != nil { - return &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} + return nil, convertAWSErr(err) } - return nil -} -func createSecretId(key secrets.SecretKey) string { - if key.Prefix == "" { - return key.Name + var secret map[string]interface{} + if err := json.Unmarshal([]byte(*result.SecretString), &secret); err != nil { + return nil, fmt.Errorf("failed to unmarshal secret value: %v", err) } - return fmt.Sprintf("%s/%s", key.Prefix, key.Name) + return secret, nil } -func init() { - if err := secrets.Register(Name, func(secretConfig map[string]interface{}) (secrets.Secrets, error) { - return New(secretConfig) - }); err != nil { - panic(err.Error()) - } +func createSecretId(key secrets.SecretKey) string { + return fmt.Sprintf("%s/%s", key.Namespace, key.Name) +} - if err := secrets.RegisterStore(Name, func(secretConfig map[string]interface{}) (secrets.SecretStore, error) { - return New(secretConfig) - }); err != nil { - panic(err.Error()) +func convertAWSErr(err error) error { + if awsErr, ok := err.(awserr.Error); ok { + return fmt.Errorf("AWS error: %s - %s", awsErr.Code(), awsErr.Message()) } + return err } + +func getSecretRetentionPeriodFromContext(ctx context.Context) int { + retentionPeriodInDays, _ := ctx.Value(SecretRetentionPeriodInDaysKey).(int) + return retentionPeriodInDays +} \ No newline at end of file diff --git a/aws/credentials/credentials.go b/aws/credentials/credentials.go index c9f5e998..1e9f306d 100644 --- a/aws/credentials/credentials.go +++ b/aws/credentials/credentials.go @@ -1,14 +1,14 @@ package credentials import ( + "context" "net/http" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go-v2/config" ) type AWSCredentials interface { @@ -22,27 +22,31 @@ type awsCred struct { func NewAWSCredentials(id, secret, token string, runningOnEc2 bool) (AWSCredentials, error) { var creds *credentials.Credentials if id != "" && secret != "" { - creds = credentials.NewStaticCredentials(id, secret, token) - if _, err := creds.Get(); err != nil { + creds = credentials.NewStaticCredentialsProvider(id, secret, token) + if _, err := creds.Retrieve(context.TODO()); err != nil { return nil, err } } else { + cfg, err := external.LoadDefaultAWSConfig() + if err != nil { + return nil, err + } providers := []credentials.Provider{ &credentials.EnvProvider{}, } if runningOnEc2 { - client := http.Client{Timeout: time.Second * 10} - sess := session.Must(session.NewSession()) + cfg.HTTPClient = &http.Client{Timeout: time.Second * 10} ec2RoleProvider := &ec2rolecreds.EC2RoleProvider{ - Client: ec2metadata.New(sess, &aws.Config{ - HTTPClient: &client, - }), + Client: ec2metadata.New(cfg), } providers = append(providers, ec2RoleProvider) } - providers = append(providers, &credentials.SharedCredentialsProvider{}) + providers = append(providers, &credentials.SharedCredentialsProvider{ + Filename: aws.StringValue(cfg.Credentials.SharedConfigFilename), + Profile: aws.StringValue(cfg.Credentials.Profile), + }) creds = credentials.NewChainCredentials(providers) - if _, err := creds.Get(); err != nil { + if _, err := creds.Retrieve(context.TODO()); err != nil { return nil, err } } @@ -50,12 +54,11 @@ func NewAWSCredentials(id, secret, token string, runningOnEc2 bool) (AWSCredenti } func (a *awsCred) Get() (*credentials.Credentials, error) { - if a.creds.IsExpired() { + if a.creds.HasExpired() { // Refresh the credentials - _, err := a.creds.Get() - if err != nil { + if _, err := a.creds.Retrieve(context.TODO()); err != nil { return nil, err } } return a.creds, nil -} +} \ No newline at end of file diff --git a/go.mod b/go.mod index 76c64bd4..082c7a9b 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,11 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.18.1 + github.com/aws/aws-sdk-go-v2/config v1.6.0 + github.com/aws/aws-sdk-go-v2/credentials v1.3.2 + github.com/aws/aws-sdk-go-v2/service/kms v1.22.1 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.9 github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect diff --git a/go.sum b/go.sum index ca0cb79f..c1f8122e 100644 --- a/go.sum +++ b/go.sum @@ -261,8 +261,9 @@ github.com/aws/aws-sdk-go v1.43.8/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4o github.com/aws/aws-sdk-go v1.44.95/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.164 h1:qDj0RutF2Ut0HZYyUJxFdReLxpYrjupsu2JmDIgCvX8= github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go-v2 v1.8.0 h1:HcN6yDnHV9S7D69E7To0aUppJhiJNEzQSNcUxc7r3qo= github.com/aws/aws-sdk-go-v2 v1.8.0/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= +github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= +github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.6.0 h1:rtoCnNObhVm7me+v9sA2aY+NtHNZjjWWC3ifXVci+wE= github.com/aws/aws-sdk-go-v2/config v1.6.0/go.mod h1:TNtBVmka80lRPk5+S9ZqVfFszOQAGJJ9KbT3EM3CHNU= github.com/aws/aws-sdk-go-v2/credentials v1.3.2 h1:Uud/fZzm0lqqhE8kvXYJFAJ3PGnagKoUcvHq1hXfBZw= @@ -271,6 +272,10 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.0 h1:SGqDJun6tydgsSIFxv9+EYBJ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.0/go.mod h1:Mj/U8OpDbcVcoctrYwA2bak8k/HFPdcLzI/vaiXMwuM= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.0 h1:Iqp2aHeRF3kaaNuDS82bHBzER285NM6lLPAgsxHCR2A= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.0/go.mod h1:eHwXu2+uE/T6gpnYWwBwqoeqRf9IXyCcolyOWDRAErQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.0 h1:xu45foJnwMwBqSkIMKyJP9kbyHi5hdhZ/WiJ7D2sHZ0= github.com/aws/aws-sdk-go-v2/internal/ini v1.2.0/go.mod h1:Q5jATQc+f1MfZp3PDMhn6ry18hGvE0i8yvbXoKbnZaE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU= @@ -279,14 +284,19 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.2 h1:Xv1rGYgsRR github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.2/go.mod h1:NXmNI41bdEsJMrD0v9rUvbGCB5GwdBEpKvUvIY3vTFg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.2 h1:ewIpdVz12MDinJJB/nu1uUiFIWFnvtd3iV7cEW7lR+M= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.2/go.mod h1:QuL2Ym8BkrLmN4lUofXYq6000/i5jPjosCNK//t6gak= +github.com/aws/aws-sdk-go-v2/service/kms v1.22.1 h1:JV2csVfMP5pxbVG1DIvRKGdW+x/K0CNLcvZoo5lOdqE= +github.com/aws/aws-sdk-go-v2/service/kms v1.22.1/go.mod h1:aNfh11Smy55o65PB3MyKbkM8BFyFUcZmj1k+4g8eNfg= github.com/aws/aws-sdk-go-v2/service/s3 v1.12.0 h1:cxZbzTYXgiQrZ6u2/RJZAkkgZssqYOdydvJPBgIHlsM= github.com/aws/aws-sdk-go-v2/service/s3 v1.12.0/go.mod h1:6J++A5xpo7QDsIeSqPK4UHqMSyPOCopa+zKtqAMhqVQ= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.9 h1:cLMyxvdb04nboUtwJl8GgyAbA25Hu7dYE/4/DV+Ku9M= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.9/go.mod h1:ezn6mzIRqTPdAbDpm03dx4y9g6rvGRb2q33wS76dCxw= github.com/aws/aws-sdk-go-v2/service/sso v1.3.2 h1:b+U3WrF9ON3f32FH19geqmiod4uKcMv/q+wosQjjyyM= github.com/aws/aws-sdk-go-v2/service/sso v1.3.2/go.mod h1:J21I6kF+d/6XHVk7kp/cx9YVD2TMD2TbLwtRGVcinXo= github.com/aws/aws-sdk-go-v2/service/sts v1.6.1 h1:1Pls85C5CFjhE3aH+h85/hyAk89kQNlAWlEQtIkaFyc= github.com/aws/aws-sdk-go-v2/service/sts v1.6.1/go.mod h1:hLZ/AnkIKHLuPGjEiyghNEdvJ2PP0MgOxcmv9EBJ4xs= -github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg= github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/axiomhq/hyperloglog v0.0.0-20220105174342-98591331716a h1:eqjiAL3qooftPm8b9C1GsSSRcmlw7iOva8vdBTmV2PY= github.com/axiomhq/hyperloglog v0.0.0-20220105174342-98591331716a/go.mod h1:2stgcRjl6QmW+gU2h5E7BQXg4HU0gzxKWDuT5HviN9s= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= From e1853a07dcbb29d49df99e6ee1c0a5eacc56a6fb Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Wed, 14 Jun 2023 19:39:08 +0530 Subject: [PATCH 02/16] DS-5114-Draft changes --- aws/aws_kms/aws_kms.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/aws/aws_kms/aws_kms.go b/aws/aws_kms/aws_kms.go index 0a1b7c22..cf38b75c 100644 --- a/aws/aws_kms/aws_kms.go +++ b/aws/aws_kms/aws_kms.go @@ -17,7 +17,25 @@ import ( "github.com/portworx/kvdb" ) -// ... Existing constants and types +const ( + // Name of the secret store + Name = secrets.TypeAWSKMS + // AwsCMKey defines the KMS customer master key + AwsCMKey = "AWS_CMK" + // KMSKvdbKey is used to setup AWS KMS Secret Store with kvdb for persistence. + KMSKvdbKey = "KMS_KVDB" + kvdbPublicBasePath = "aws_kms/secrets/public/" + kvdbDataBasePath = "aws_kms/secrets/data/" +) + +type awsKmsSecrets struct { + client *kms.KMS + creds *credentials.Credentials + sess *session.Session + cmk string + asc sc.AWSCredentials + ps store.PersistenceStore +} func New( secretConfig map[string]interface{}, From 795621a3478879cd3ac2e910c58b395c9b04a822 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Wed, 14 Jun 2023 19:43:24 +0530 Subject: [PATCH 03/16] DS-5114-Draft changes --- aws/aws_secrets_manager/aws_scm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/aws_secrets_manager/aws_scm.go b/aws/aws_secrets_manager/aws_scm.go index a5265617..23a7dafb 100644 --- a/aws/aws_secrets_manager/aws_scm.go +++ b/aws/aws_secrets_manager/aws_scm.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/libopenstorage/secrets" sc "github.com/libopenstorage/secrets/aws/credentials" From 6d3bf6a3032c5c942fb258f1acf2e9b541d124c6 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Wed, 14 Jun 2023 20:27:16 +0530 Subject: [PATCH 04/16] Revert "Update minikube setup versions (#74)" This reverts commit 1022cc4d5aeb8bceedfc664b32667755b35e6a15. --- .github/workflows/canary-integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index f06626ab..f3d5a068 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -25,8 +25,8 @@ jobs: - name: setup minikube uses: manusa/actions-setup-minikube@v2.4.2 with: - minikube version: 'v1.30.1' - kubernetes version: 'v1.22.17' + minikube version: 'v1.21.0' + kubernetes version: 'v1.19.2' start args: --memory 6g --cpus=2 github token: ${{ secrets.GITHUB_TOKEN }} From b29a619ddf52ddfcea02658684594d66cb242328 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Wed, 14 Jun 2023 20:27:50 +0530 Subject: [PATCH 05/16] Revert "Revert "Update minikube setup versions (#74)"" This reverts commit 6d3bf6a3032c5c942fb258f1acf2e9b541d124c6. --- .github/workflows/canary-integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index f3d5a068..f06626ab 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -25,8 +25,8 @@ jobs: - name: setup minikube uses: manusa/actions-setup-minikube@v2.4.2 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.19.2' + minikube version: 'v1.30.1' + kubernetes version: 'v1.22.17' start args: --memory 6g --cpus=2 github token: ${{ secrets.GITHUB_TOKEN }} From 0304ae7ea03c8d4efb7e92b3cc8a67a13992f363 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Wed, 21 Jun 2023 21:21:36 +0530 Subject: [PATCH 06/16] Updated code to support AWS SDK V2 --- aws/aws_kms/aws_kms.go | 63 ++++--- aws/aws_secrets_manager/aws_scm.go | 292 ++++++++++++++++++----------- aws/credentials/credentials.go | 64 +++---- go.mod | 10 +- go.sum | 25 ++- 5 files changed, 274 insertions(+), 180 deletions(-) diff --git a/aws/aws_kms/aws_kms.go b/aws/aws_kms/aws_kms.go index cf38b75c..edcd629c 100644 --- a/aws/aws_kms/aws_kms.go +++ b/aws/aws_kms/aws_kms.go @@ -4,10 +4,11 @@ import ( "context" "encoding/base64" "fmt" - "github.com/libopenstorage/secrets/aws/utils" "os" - "github.com/aws/aws-sdk-go-v2/config" + "github.com/libopenstorage/secrets/aws/utils" + + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" @@ -29,9 +30,8 @@ const ( ) type awsKmsSecrets struct { - client *kms.KMS - creds *credentials.Credentials - sess *session.Session + client *kms.Client + creds *aws.Credentials cmk string asc sc.AWSCredentials ps store.PersistenceStore @@ -85,25 +85,14 @@ func New( if err != nil { return nil, fmt.Errorf("Failed to get credentials: %v", err) } - - cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ - Value: aws.Credentials{ - AccessKeyID: creds.AccessKeyID, - SecretAccessKey: creds.SecretAccessKey, - SessionToken: creds.SessionToken, - Source: creds.Source, - CanExpire: creds.CanExpire, - Expires: creds.Expires, - }, - }), - config.WithRegion(region), - ) - if err != nil { - return nil, err + credProv := credentialsToProvider(creds) + config := aws.Config{ + Credentials: credProv, + Region: region, } - kmsClient := kms.NewFromConfig(cfg) + kmsClient := kms.NewFromConfig(config) + return &awsKmsSecrets{ client: kmsClient, creds: creds, @@ -113,6 +102,21 @@ func New( }, nil } +func credentialsToProvider(creds *aws.Credentials) aws.CredentialsProvider { + return credentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: creds.AccessKeyID, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + Source: creds.Source, + }, + } +} + +func (a *awsKmsSecrets) String() string { + return Name +} + func (a *awsKmsSecrets) GetSecret( secretId string, keyContext map[string]string, @@ -141,14 +145,14 @@ func (a *awsKmsSecrets) GetSecret( return secretData, secrets.NoVersion, nil } - // AWS KMS API requires the cipherBlob to be in base64 decoded format. + // AWS KMS api requires the cipherBlob to be in base64 decoded format. // Check if it is encoded and decode if required. decodedCipherBlob, err = base64.StdEncoding.DecodeString(string(cipherBlob)) if err != nil { decodedCipherBlob = cipherBlob } input := &kms.DecryptInput{ - EncryptionContext: getAWSKeyContext(keyContext), + EncryptionContext: keyContext, CiphertextBlob: decodedCipherBlob, } output, err := a.client.Decrypt(context.TODO(), input) @@ -156,7 +160,7 @@ func (a *awsKmsSecrets) GetSecret( return nil, secrets.NoVersion, err } - // filePersistenceStore does not support storing secretData + // filePersistenceStore does not support storing of secretData if a.ps.Name() == store.FilePersistenceStoreName { goto return_plaintext } @@ -181,6 +185,7 @@ func (a *awsKmsSecrets) PutSecret( secretData map[string]interface{}, keyContext map[string]string, ) (secrets.Version, error) { + _, override := keyContext[secrets.OverwriteSecretDataInStore] _, publicData := keyContext[secrets.PublicSecretData] @@ -208,11 +213,11 @@ func (a *awsKmsSecrets) PutSecret( ) } - keySpec := types.DataKeySpecAes256 + keySpec := "AES_256" input := &kms.GenerateDataKeyInput{ KeyId: &a.cmk, - EncryptionContext: getAWSKeyContext(keyContext), - KeySpec: keySpec, + EncryptionContext: keyContext, + KeySpec: types.DataKeySpec(keySpec), } output, err := a.client.GenerateDataKey(context.TODO(), input) @@ -287,4 +292,4 @@ func init() { if err := secrets.Register(Name, New); err != nil { panic(err.Error()) } -} \ No newline at end of file +} diff --git a/aws/aws_secrets_manager/aws_scm.go b/aws/aws_secrets_manager/aws_scm.go index 23a7dafb..d85c70b6 100644 --- a/aws/aws_secrets_manager/aws_scm.go +++ b/aws/aws_secrets_manager/aws_scm.go @@ -3,13 +3,16 @@ package aws_secrets_manager import ( "context" "encoding/json" + "errors" "fmt" "os" "strconv" "strings" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/smithy-go" "github.com/libopenstorage/secrets" sc "github.com/libopenstorage/secrets/aws/credentials" "github.com/libopenstorage/secrets/aws/utils" @@ -37,7 +40,7 @@ func New( awsConfig, ok := secretConfig[utils.AwsConfigKey] if ok { - awsConfig, ok := awsConfig.(*aws.Config) + awsConfig, ok := awsConfig.(aws.Config) if !ok { return nil, utils.ErrAWSConfigWrongType } @@ -66,22 +69,30 @@ func New( if err != nil { return nil, fmt.Errorf("failed to get credentials: %v", err) } + credProv := CredentialsToProvider(creds) config := aws.Config{ - Credentials: creds, + Credentials: credProv, Region: region, } - return NewFromAWSConfig(&config) + return NewFromAWSConfig(config) } -// NewFromAWSConfig creates new instance of AWSSecretsMgr with provided AWS configuration (aws.Config). -func NewFromAWSConfig(config *aws.Config) (*AWSSecretsMgr, error) { - cfg, err := external.LoadDefaultAWSConfig(*config) - if err != nil { - return nil, fmt.Errorf("failed to load AWS config: %v", err) +// credentialsToProvider converts a aws.Credential object to a aws.CredentialProvider object +func CredentialsToProvider(creds *aws.Credentials) aws.CredentialsProvider { + return credentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: creds.AccessKeyID, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + Source: creds.Source, + }, } +} - scm := secretsmanager.NewFromConfig(cfg) +// NewFromAWSConfig creates new instance of AWSSecretsMgr with provided AWS configuration (aws.Config). +func NewFromAWSConfig(config aws.Config) (*AWSSecretsMgr, error) { + scm := secretsmanager.NewFromConfig(config) return &AWSSecretsMgr{ scm: scm, }, nil @@ -103,140 +114,209 @@ func (a *AWSSecretsMgr) Set(_ context.Context, key secrets.SecretKey, secret map return err } -func (a *AWSSecretsMgr) Delete(ctx context.Context, key secrets.SecretKey) error { +func (a *AWSSecretsMgr) Delete(_ context.Context, key secrets.SecretKey) error { secretID := createSecretId(key) - - retentionPeriodInDays := getSecretRetentionPeriodFromContext(ctx) - - if retentionPeriodInDays > 0 { - return a.deleteSecretWithRetention(secretID, retentionPeriodInDays) - } - - _, err := a.deleteSecret(secretID) - return err + return a.delete(secretID, 0) } -func (a *AWSSecretsMgr) get(secretID string) (map[string]interface{}, string, error) { - input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(secretID), - } +func (a *AWSSecretsMgr) GetSecret( + secretID string, + _ map[string]string, +) (map[string]interface{}, secrets.Version, error) { + return a.get(secretID) +} - result, err := a.scm.GetSecretValue(context.TODO(), input) - if err != nil { - return nil, "", convertAWSErr(err) - } +func (a *AWSSecretsMgr) PutSecret( + secretID string, + secretData map[string]interface{}, + _ map[string]string, +) (secrets.Version, error) { + return a.put(secretID, secretData) +} - var secret map[string]interface{} - if err := json.Unmarshal([]byte(*result.SecretString), &secret); err != nil { - return nil, "", fmt.Errorf("failed to unmarshal secret value: %v", err) +func (a *AWSSecretsMgr) DeleteSecret( + secretID string, + keyContext map[string]string, +) error { + retentionPeriodInDays := 0 + retentionPeriod, ok := keyContext[SecretRetentionPeriodInDaysKey] + if ok { + var err error + retentionPeriodInDays, err = strconv.Atoi(retentionPeriod) + if err != nil { + return &secrets.ErrProviderInternal{ + Reason: "invalid retention period, value must be a number between 7 and 30", + Provider: Name, + } + } } - return secret, *result.VersionId, nil + return a.delete(secretID, int64(retentionPeriodInDays)) } -func (a *AWSSecretsMgr) put(secretID string, secret map[string]interface{}) (string, error) { - secretValue, err := json.Marshal(secret) - if err != nil { - return "", fmt.Errorf("failed to marshal secret value: %v", err) - } +func (a *AWSSecretsMgr) ListSecrets() ([]string, error) { + return nil, secrets.ErrNotSupported +} - input := &secretsmanager.PutSecretValueInput{ - SecretId: aws.String(secretID), - SecretString: aws.String(string(secretValue)), - } +func (a *AWSSecretsMgr) Encrypt( + _ string, + _ string, + _ map[string]string, +) (string, error) { + return "", secrets.ErrNotSupported +} - result, err := a.scm.PutSecretValue(context.TODO(), input) - if err != nil { - return "", convertAWSErr(err) - } +func (a *AWSSecretsMgr) Decrypt( + _ string, + _ string, + _ map[string]string, +) (string, error) { + return "", secrets.ErrNotSupported +} - return *result.VersionId, nil +func (a *AWSSecretsMgr) Rencrypt( + _ string, + _ string, + _ map[string]string, + _ map[string]string, + _ string, +) (string, error) { + return "", secrets.ErrNotSupported } -func (a *AWSSecretsMgr) deleteSecret(secretID string) (string, error) { - input := &secretsmanager.DeleteSecretInput{ +func (a *AWSSecretsMgr) get(secretID string) (map[string]interface{}, secrets.Version, error) { + secretValueOutput, err := a.scm.GetSecretValue(context.TODO(), &secretsmanager.GetSecretValueInput{ SecretId: aws.String(secretID), - } - - result, err := a.scm.DeleteSecret(context.TODO(), input) + }) if err != nil { - return "", convertAWSErr(err) + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + // aerr, ok := err.(awserr.Error); ok { + if apiErr.ErrorCode() == "ResourceNotFoundException" { + return nil, secrets.NoVersion, secrets.ErrInvalidSecretId + } else if apiErr.ErrorCode() == "InvalidRequestException" && + strings.Contains(apiErr.ErrorCode(), "Marked for deletion") { + return nil, secrets.NoVersion, secrets.ErrInvalidSecretId + } + } + return nil, secrets.NoVersion, &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} } + if secretValueOutput.SecretString == nil || len(*secretValueOutput.SecretString) <= 0 { + return nil, secrets.NoVersion, secrets.ErrEmptySecretData + } + secretOut := make(map[string]interface{}) + if err := json.Unmarshal([]byte(*secretValueOutput.SecretString), &secretOut); err != nil { + secretOut = make(map[string]interface{}) + secretOut[secretID] = *secretValueOutput.SecretString - return *result.DeletionDate, nil -} - -func (a *AWSSecretsMgr) deleteSecretWithRetention(secretID string, retentionPeriodInDays int) error { - input := &secretsmanager.DeleteSecretInput{ - SecretId: aws.String(secretID), } - ctx := context.WithValue(context.Background(), SecretRetentionPeriodInDaysKey, retentionPeriodInDays) - _, err := a.scm.DeleteSecretWithContext(ctx, input) - if err != nil { - return convertAWSErr(err) + if secretValueOutput.VersionId == nil { + return nil, secrets.NoVersion, fmt.Errorf("invalid version returned by aws") } - - return nil + return secretOut, secrets.Version(*secretValueOutput.VersionId), nil } -func (a *AWSSecretsMgr) Type() string { - return Name -} - -func (a *AWSSecretsMgr) Versions(key secrets.SecretKey) ([]string, error) { - secretID := createSecretId(key) - - input := &secretsmanager.DescribeSecretInput{ - SecretId: aws.String(secretID), - } - - result, err := a.scm.DescribeSecret(context.Background(), input) +func (a *AWSSecretsMgr) put( + secretID string, + secretData map[string]interface{}, +) (secrets.Version, error) { + // Marshal the secret data + secretBytes, err := json.Marshal(secretData) if err != nil { - return nil, convertAWSErr(err) + return secrets.NoVersion, fmt.Errorf("failed to marshal secret data: %v", err) } - - var versions []string - for _, version := range result.VersionIdsToStages { - versions = append(versions, *version) + // Check if there already exists a key. + _, err = a.scm.GetSecretValue(context.TODO(), &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretID), + }) + if err == nil { + // Update the existing secret + secretValueOutput, putErr := a.scm.PutSecretValue(context.TODO(), &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(secretID), + SecretString: aws.String(string(secretBytes)), + }) + if putErr != nil { + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: putErr.Error(), Provider: Name} + } + if secretValueOutput.VersionId == nil { + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: "invalid version returned by aws", Provider: Name} + } + return secrets.Version(*secretValueOutput.VersionId), nil + } else { + // if aerr, ok := err.(awserr.Error); ok { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + if apiErr.ErrorCode() == "ResourceNotFoundException" { + // Create a new secret + secretValueOutput, createErr := a.scm.CreateSecret(context.TODO(), &secretsmanager.CreateSecretInput{ + SecretString: aws.String(string(secretBytes)), + Name: aws.String(secretID), + }) + if createErr != nil { + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: createErr.Error(), Provider: Name} + } + if secretValueOutput.VersionId == nil { + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: "invalid version returned by aws", Provider: Name} + } + return secrets.Version(*secretValueOutput.VersionId), nil + } // return the aws error + } // return the non-aws error } - - return versions, nil + // Gets, Puts & Creates have failed + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} } -func (a *AWSSecretsMgr) Version(key secrets.SecretKey, version string) (map[string]interface{}, error) { - secretID := createSecretId(key) +func (a *AWSSecretsMgr) delete( + secretID string, + retentionPeriodInDays int64, +) error { + if retentionPeriodInDays != 0 && (retentionPeriodInDays < 7 || retentionPeriodInDays > 30) { + return &secrets.ErrProviderInternal{ + Reason: "invalid retention period, value must be a number between 7 and 30", + Provider: Name, + } + } - input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(secretID), - VersionStage: aws.String(version), - VersionId: aws.String(version), + var deleteSecretInput *secretsmanager.DeleteSecretInput + if retentionPeriodInDays == 0 { + deleteSecretInput = &secretsmanager.DeleteSecretInput{ + // By default, aws keeps the secret for 30 days + // Delete immediately + ForceDeleteWithoutRecovery: aws.Bool(true), + SecretId: aws.String(secretID), + } + } else { + deleteSecretInput = &secretsmanager.DeleteSecretInput{ + SecretId: aws.String(secretID), + RecoveryWindowInDays: aws.Int64(retentionPeriodInDays), + } } - result, err := a.scm.GetSecretValue(context.Background(), input) + _, err := a.scm.DeleteSecret(context.TODO(), deleteSecretInput) if err != nil { - return nil, convertAWSErr(err) + return &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} } + return nil +} - var secret map[string]interface{} - if err := json.Unmarshal([]byte(*result.SecretString), &secret); err != nil { - return nil, fmt.Errorf("failed to unmarshal secret value: %v", err) +func createSecretId(key secrets.SecretKey) string { + if key.Prefix == "" { + return key.Name } - return secret, nil + return fmt.Sprintf("%s/%s", key.Prefix, key.Name) } -func createSecretId(key secrets.SecretKey) string { - return fmt.Sprintf("%s/%s", key.Namespace, key.Name) -} +func init() { + if err := secrets.Register(Name, func(secretConfig map[string]interface{}) (secrets.Secrets, error) { + return New(secretConfig) + }); err != nil { + panic(err.Error()) + } -func convertAWSErr(err error) error { - if awsErr, ok := err.(awserr.Error); ok { - return fmt.Errorf("AWS error: %s - %s", awsErr.Code(), awsErr.Message()) + if err := secrets.RegisterStore(Name, func(secretConfig map[string]interface{}) (secrets.SecretStore, error) { + return New(secretConfig) + }); err != nil { + panic(err.Error()) } - return err } - -func getSecretRetentionPeriodFromContext(ctx context.Context) int { - retentionPeriodInDays, _ := ctx.Value(SecretRetentionPeriodInDaysKey).(int) - return retentionPeriodInDays -} \ No newline at end of file diff --git a/aws/credentials/credentials.go b/aws/credentials/credentials.go index 1e9f306d..c2ea8fac 100644 --- a/aws/credentials/credentials.go +++ b/aws/credentials/credentials.go @@ -2,63 +2,63 @@ package credentials import ( "context" - "net/http" + "log" "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" ) type AWSCredentials interface { - Get() (*credentials.Credentials, error) + Get() (*aws.Credentials, error) } type awsCred struct { - creds *credentials.Credentials + creds *aws.Credentials } func NewAWSCredentials(id, secret, token string, runningOnEc2 bool) (AWSCredentials, error) { - var creds *credentials.Credentials + var creds aws.Credentials if id != "" && secret != "" { - creds = credentials.NewStaticCredentialsProvider(id, secret, token) - if _, err := creds.Retrieve(context.TODO()); err != nil { + provider := credentials.NewStaticCredentialsProvider(id, secret, token) + var err error + if creds, err = provider.Retrieve(context.TODO()); err != nil { return nil, err } } else { - cfg, err := external.LoadDefaultAWSConfig() + defaultCfg, err := config.LoadDefaultConfig(context.TODO()) if err != nil { return nil, err } - providers := []credentials.Provider{ - &credentials.EnvProvider{}, - } if runningOnEc2 { - cfg.HTTPClient = &http.Client{Timeout: time.Second * 10} - ec2RoleProvider := &ec2rolecreds.EC2RoleProvider{ - Client: ec2metadata.New(cfg), + defaultCfg.HTTPClient = http.NewBuildableClient().WithTimeout(10 * time.Second) + + defaultProvider := config.WithCredentialsProvider(defaultCfg.Credentials) + ec2provider := config.WithCredentialsProvider(ec2rolecreds.New(func(o *ec2rolecreds.Options) { + o.Client = imds.NewFromConfig(defaultCfg) + })) + + cfg, err := config.LoadDefaultConfig(context.TODO(), + defaultProvider, + ec2provider, + ) + if err != nil { + log.Fatal(err) + } + + creds, err = cfg.Credentials.Retrieve(context.Background()) + if err != nil { + return nil, err } - providers = append(providers, ec2RoleProvider) - } - providers = append(providers, &credentials.SharedCredentialsProvider{ - Filename: aws.StringValue(cfg.Credentials.SharedConfigFilename), - Profile: aws.StringValue(cfg.Credentials.Profile), - }) - creds = credentials.NewChainCredentials(providers) - if _, err := creds.Retrieve(context.TODO()); err != nil { - return nil, err } } - return &awsCred{creds}, nil + return &awsCred{&creds}, nil } -func (a *awsCred) Get() (*credentials.Credentials, error) { - if a.creds.HasExpired() { - // Refresh the credentials - if _, err := a.creds.Retrieve(context.TODO()); err != nil { - return nil, err - } - } +func (a *awsCred) Get() (*aws.Credentials, error) { return a.creds, nil -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 082c7a9b..e6bac0a2 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/IBM/keyprotect-go-client v0.5.1 - github.com/aws/aws-sdk-go v1.44.164 + github.com/aws/aws-sdk-go v1.44.164 // indirect github.com/golang/mock v1.6.0 github.com/hashicorp/go-hclog v1.3.1 github.com/hashicorp/vault v1.12.2 @@ -27,9 +27,11 @@ require ( require ( github.com/aws/aws-sdk-go-v2 v1.18.1 github.com/aws/aws-sdk-go-v2/config v1.6.0 - github.com/aws/aws-sdk-go-v2/credentials v1.3.2 - github.com/aws/aws-sdk-go-v2/service/kms v1.22.1 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.9 + github.com/aws/aws-sdk-go-v2/credentials v1.13.26 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 + github.com/aws/aws-sdk-go-v2/service/kms v1.22.2 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.10 + github.com/aws/smithy-go v1.13.5 github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect diff --git a/go.sum b/go.sum index c1f8122e..c931d7d0 100644 --- a/go.sum +++ b/go.sum @@ -266,10 +266,12 @@ github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCe github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2/config v1.6.0 h1:rtoCnNObhVm7me+v9sA2aY+NtHNZjjWWC3ifXVci+wE= github.com/aws/aws-sdk-go-v2/config v1.6.0/go.mod h1:TNtBVmka80lRPk5+S9ZqVfFszOQAGJJ9KbT3EM3CHNU= -github.com/aws/aws-sdk-go-v2/credentials v1.3.2 h1:Uud/fZzm0lqqhE8kvXYJFAJ3PGnagKoUcvHq1hXfBZw= github.com/aws/aws-sdk-go-v2/credentials v1.3.2/go.mod h1:PACKuTJdt6AlXvEq8rFI4eDmoqDFC5DpVKQbWysaDgM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.0 h1:SGqDJun6tydgsSIFxv9+EYBJVqVUwg2QMJp6PbNq8C8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.0/go.mod h1:Mj/U8OpDbcVcoctrYwA2bak8k/HFPdcLzI/vaiXMwuM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.0 h1:Iqp2aHeRF3kaaNuDS82bHBzER285NM6lLPAgsxHCR2A= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.4.0/go.mod h1:eHwXu2+uE/T6gpnYWwBwqoeqRf9IXyCcolyOWDRAErQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= @@ -280,20 +282,25 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.2.0 h1:xu45foJnwMwBqSkIMKyJP9kbyHi5 github.com/aws/aws-sdk-go-v2/internal/ini v1.2.0/go.mod h1:Q5jATQc+f1MfZp3PDMhn6ry18hGvE0i8yvbXoKbnZaE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.2 h1:Xv1rGYgsRRn0xw9JFNnfpBMZam54PrWpC4rJOJ9koA8= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.2/go.mod h1:NXmNI41bdEsJMrD0v9rUvbGCB5GwdBEpKvUvIY3vTFg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.2 h1:ewIpdVz12MDinJJB/nu1uUiFIWFnvtd3iV7cEW7lR+M= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.2/go.mod h1:QuL2Ym8BkrLmN4lUofXYq6000/i5jPjosCNK//t6gak= -github.com/aws/aws-sdk-go-v2/service/kms v1.22.1 h1:JV2csVfMP5pxbVG1DIvRKGdW+x/K0CNLcvZoo5lOdqE= -github.com/aws/aws-sdk-go-v2/service/kms v1.22.1/go.mod h1:aNfh11Smy55o65PB3MyKbkM8BFyFUcZmj1k+4g8eNfg= +github.com/aws/aws-sdk-go-v2/service/kms v1.22.2 h1:jwmtdM1/l1DRNy5jQrrYpsQm8zwetkgeqhAqefDr1yI= +github.com/aws/aws-sdk-go-v2/service/kms v1.22.2/go.mod h1:aNfh11Smy55o65PB3MyKbkM8BFyFUcZmj1k+4g8eNfg= github.com/aws/aws-sdk-go-v2/service/s3 v1.12.0 h1:cxZbzTYXgiQrZ6u2/RJZAkkgZssqYOdydvJPBgIHlsM= github.com/aws/aws-sdk-go-v2/service/s3 v1.12.0/go.mod h1:6J++A5xpo7QDsIeSqPK4UHqMSyPOCopa+zKtqAMhqVQ= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.9 h1:cLMyxvdb04nboUtwJl8GgyAbA25Hu7dYE/4/DV+Ku9M= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.9/go.mod h1:ezn6mzIRqTPdAbDpm03dx4y9g6rvGRb2q33wS76dCxw= -github.com/aws/aws-sdk-go-v2/service/sso v1.3.2 h1:b+U3WrF9ON3f32FH19geqmiod4uKcMv/q+wosQjjyyM= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.10 h1:eW8zPSh7ZLzb7029xCsIEFbnxLvNHPTt7aWwdKjNJc8= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.19.10/go.mod h1:ezn6mzIRqTPdAbDpm03dx4y9g6rvGRb2q33wS76dCxw= github.com/aws/aws-sdk-go-v2/service/sso v1.3.2/go.mod h1:J21I6kF+d/6XHVk7kp/cx9YVD2TMD2TbLwtRGVcinXo= -github.com/aws/aws-sdk-go-v2/service/sts v1.6.1 h1:1Pls85C5CFjhE3aH+h85/hyAk89kQNlAWlEQtIkaFyc= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.12/go.mod h1:HuCOxYsF21eKrerARYO6HapNeh9GBNq7fius2AcwodY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9Xthr0zVFTgBrmPldILn80= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w= github.com/aws/aws-sdk-go-v2/service/sts v1.6.1/go.mod h1:hLZ/AnkIKHLuPGjEiyghNEdvJ2PP0MgOxcmv9EBJ4xs= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg= github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= From 78bf6450b80bfba4550f3dfb369375c3399e9ba0 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 16:47:28 +0530 Subject: [PATCH 07/16] Changed retention period to 7 days for integration test --- aws/aws_secrets_manager/aws_scm_integration_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index 8f3ba526..a25f096b 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -107,7 +107,10 @@ func (a *awsSecretTest) TestListSecrets(t *testing.T) error { func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { // Delete of a key that exists should succeed - err := a.s.DeleteSecret(a.secretIdWithData, nil) + keyContext := make(map[string]string) + keyContext[SecretRetentionPeriodInDaysKey] = "7" + + err := a.s.DeleteSecret(a.secretIdWithData, keyContext) assert.NoError(t, err, "Expected DeleteSecret to succeed") // Get of a deleted key should fail From e0fa47462f722fbbad355b3688b2052346a39a43 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 17:02:08 +0530 Subject: [PATCH 08/16] Add a delay to allow time for deletion to propagate --- aws/aws_kms/aws_kms_integration_test.go | 4 ++++ aws/aws_secrets_manager/aws_scm_integration_test.go | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/aws/aws_kms/aws_kms_integration_test.go b/aws/aws_kms/aws_kms_integration_test.go index 92a408e0..19a7b693 100644 --- a/aws/aws_kms/aws_kms_integration_test.go +++ b/aws/aws_kms/aws_kms_integration_test.go @@ -6,6 +6,7 @@ package aws_kms import ( "os" "testing" + "time" "github.com/libopenstorage/secrets" "github.com/libopenstorage/secrets/aws/utils" @@ -164,6 +165,9 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { err := a.s.DeleteSecret(secretIdWithData, nil) assert.NoError(t, err, "Expected DeleteSecret to succeed") + // Add a delay to allow time for deletion to propagate + time.Sleep(time.Second * 30) + // Get of a deleted key should fail _, _, err = a.s.GetSecret(secretIdWithData, nil) assert.EqualError(t, secrets.ErrInvalidSecretId, err.Error(), "Unexpected error on GetSecret after delete") diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index a25f096b..4d7749e7 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -6,6 +6,7 @@ package aws_secrets_manager import ( "os" "testing" + "time" "github.com/libopenstorage/secrets" "github.com/libopenstorage/secrets/aws/utils" @@ -113,6 +114,9 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { err := a.s.DeleteSecret(a.secretIdWithData, keyContext) assert.NoError(t, err, "Expected DeleteSecret to succeed") + // Add a delay to allow time for deletion to propagate + time.Sleep(time.Second * 30) + // Get of a deleted key should fail _, version, err := a.s.GetSecret(a.secretIdWithData, nil) assert.EqualError(t, secrets.ErrInvalidSecretId, err.Error(), "Unexpected error on GetSecret after delete") @@ -122,6 +126,9 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { err = a.s.DeleteSecret(a.secretIdWithoutData, nil) assert.NoError(t, err, "Expected DeleteSecret to succeed") + // Add a delay to allow time for deletion to propagate + time.Sleep(time.Second * 30) + // GetSecret using a secretId without data _, version, err = a.s.GetSecret(a.secretIdWithoutData, nil) assert.EqualError(t, secrets.ErrInvalidSecretId, err.Error(), "Unexpected error on GetSecret after delete") From cd3f98ab055c8784afdefa8e6468703afc33e8ac Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 17:11:08 +0530 Subject: [PATCH 09/16] Add a delay to allow time for deletion to propagate --- aws/aws_kms/aws_kms_integration_test.go | 2 +- aws/aws_secrets_manager/aws_scm_integration_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/aws_kms/aws_kms_integration_test.go b/aws/aws_kms/aws_kms_integration_test.go index 19a7b693..82ce4c76 100644 --- a/aws/aws_kms/aws_kms_integration_test.go +++ b/aws/aws_kms/aws_kms_integration_test.go @@ -166,7 +166,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 30) + time.Sleep(time.Second * 90) // Get of a deleted key should fail _, _, err = a.s.GetSecret(secretIdWithData, nil) diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index 4d7749e7..d7c41c01 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -115,7 +115,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 30) + time.Sleep(time.Second * 90) // Get of a deleted key should fail _, version, err := a.s.GetSecret(a.secretIdWithData, nil) @@ -127,7 +127,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 30) + time.Sleep(time.Second * 90) // GetSecret using a secretId without data _, version, err = a.s.GetSecret(a.secretIdWithoutData, nil) From c073ad0ad55f8dda0f76c93711f8b7854d74b485 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 17:19:16 +0530 Subject: [PATCH 10/16] Add a delay to allow time for deletion to propagate --- aws/aws_secrets_manager/aws_scm_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index d7c41c01..8478dce0 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -123,7 +123,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.Equal(t, version, secrets.NoVersion) // Delete of a key that exists should succeed - err = a.s.DeleteSecret(a.secretIdWithoutData, nil) + err = a.s.DeleteSecret(a.secretIdWithoutData, keyContext) assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate From bd0945e5a87b73ddce9a2449b6e0d01da60cb9d8 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 17:33:14 +0530 Subject: [PATCH 11/16] Add a delay to allow time for deletion to propagate --- aws/aws_secrets_manager/aws_scm_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index 8478dce0..8a47d703 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -115,7 +115,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 90) + time.Sleep(time.Second * 600) // Get of a deleted key should fail _, version, err := a.s.GetSecret(a.secretIdWithData, nil) @@ -127,7 +127,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 90) + time.Sleep(time.Second * 600) // GetSecret using a secretId without data _, version, err = a.s.GetSecret(a.secretIdWithoutData, nil) From 61157c58bd417844a8b8732ae2f50fa7d0581fe0 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 17:55:55 +0530 Subject: [PATCH 12/16] reduce sleep time --- aws/aws_secrets_manager/aws_scm_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index 8a47d703..3de99535 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -115,7 +115,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 600) + time.Sleep(time.Second * 300) // Get of a deleted key should fail _, version, err := a.s.GetSecret(a.secretIdWithData, nil) @@ -127,7 +127,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 600) + time.Sleep(time.Second * 300) // GetSecret using a secretId without data _, version, err = a.s.GetSecret(a.secretIdWithoutData, nil) From 1762ab5af46d4755cabeaabf4eaf3936d5ca2d04 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 23 Jun 2023 18:25:01 +0530 Subject: [PATCH 13/16] reduce sleep time --- aws/aws_secrets_manager/aws_scm_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/aws_secrets_manager/aws_scm_integration_test.go b/aws/aws_secrets_manager/aws_scm_integration_test.go index 3de99535..770ed964 100644 --- a/aws/aws_secrets_manager/aws_scm_integration_test.go +++ b/aws/aws_secrets_manager/aws_scm_integration_test.go @@ -115,7 +115,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 300) + time.Sleep(time.Second * 200) // Get of a deleted key should fail _, version, err := a.s.GetSecret(a.secretIdWithData, nil) @@ -127,7 +127,7 @@ func (a *awsSecretTest) TestDeleteSecret(t *testing.T) error { assert.NoError(t, err, "Expected DeleteSecret to succeed") // Add a delay to allow time for deletion to propagate - time.Sleep(time.Second * 300) + time.Sleep(time.Second * 200) // GetSecret using a secretId without data _, version, err = a.s.GetSecret(a.secretIdWithoutData, nil) From 5f953e754f247b280ae7635029629ac2020fb1e3 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Fri, 1 Sep 2023 20:19:11 +0530 Subject: [PATCH 14/16] Changed error handling by using secretmanager error handler instead of smithyAPI --- aws/aws_secrets_manager/aws_scm.go | 47 ++++++++++++------------------ 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/aws/aws_secrets_manager/aws_scm.go b/aws/aws_secrets_manager/aws_scm.go index d85c70b6..5f4cb904 100644 --- a/aws/aws_secrets_manager/aws_scm.go +++ b/aws/aws_secrets_manager/aws_scm.go @@ -3,7 +3,6 @@ package aws_secrets_manager import ( "context" "encoding/json" - "errors" "fmt" "os" "strconv" @@ -12,7 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" - "github.com/aws/smithy-go" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" "github.com/libopenstorage/secrets" sc "github.com/libopenstorage/secrets/aws/credentials" "github.com/libopenstorage/secrets/aws/utils" @@ -189,15 +188,11 @@ func (a *AWSSecretsMgr) get(secretID string) (map[string]interface{}, secrets.Ve SecretId: aws.String(secretID), }) if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - // aerr, ok := err.(awserr.Error); ok { - if apiErr.ErrorCode() == "ResourceNotFoundException" { - return nil, secrets.NoVersion, secrets.ErrInvalidSecretId - } else if apiErr.ErrorCode() == "InvalidRequestException" && - strings.Contains(apiErr.ErrorCode(), "Marked for deletion") { - return nil, secrets.NoVersion, secrets.ErrInvalidSecretId - } + if _, ok := err.(*types.ResourceNotFoundException); ok { + return nil, secrets.NoVersion, secrets.ErrInvalidSecretId + } else if aerr, ok := err.(*types.InvalidRequestException); ok && + strings.Contains(aerr.Error(), "marked for deletion") { + return nil, secrets.NoVersion, secrets.ErrInvalidSecretId } return nil, secrets.NoVersion, &secrets.ErrProviderInternal{Reason: err.Error(), Provider: Name} } @@ -243,23 +238,19 @@ func (a *AWSSecretsMgr) put( } return secrets.Version(*secretValueOutput.VersionId), nil } else { - // if aerr, ok := err.(awserr.Error); ok { - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - if apiErr.ErrorCode() == "ResourceNotFoundException" { - // Create a new secret - secretValueOutput, createErr := a.scm.CreateSecret(context.TODO(), &secretsmanager.CreateSecretInput{ - SecretString: aws.String(string(secretBytes)), - Name: aws.String(secretID), - }) - if createErr != nil { - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: createErr.Error(), Provider: Name} - } - if secretValueOutput.VersionId == nil { - return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: "invalid version returned by aws", Provider: Name} - } - return secrets.Version(*secretValueOutput.VersionId), nil - } // return the aws error + if _, ok := err.(*types.ResourceNotFoundException); ok { + // Create a new secret + secretValueOutput, createErr := a.scm.CreateSecret(context.TODO(), &secretsmanager.CreateSecretInput{ + SecretString: aws.String(string(secretBytes)), + Name: aws.String(secretID), + }) + if createErr != nil { + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: createErr.Error(), Provider: Name} + } + if secretValueOutput.VersionId == nil { + return secrets.NoVersion, &secrets.ErrProviderInternal{Reason: "invalid version returned by aws", Provider: Name} + } + return secrets.Version(*secretValueOutput.VersionId), nil } // return the non-aws error } // Gets, Puts & Creates have failed From 4c539350eb653d957f05581cc9f91b6e87b17f38 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Sun, 3 Sep 2023 20:29:59 +0530 Subject: [PATCH 15/16] Made changes to the struct --- aws/credentials/credentials.go | 56 +++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/aws/credentials/credentials.go b/aws/credentials/credentials.go index c2ea8fac..eb0c0095 100644 --- a/aws/credentials/credentials.go +++ b/aws/credentials/credentials.go @@ -2,7 +2,6 @@ package credentials import ( "context" - "log" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -18,47 +17,54 @@ type AWSCredentials interface { } type awsCred struct { - creds *aws.Credentials + creds *aws.Credentials + credsprovider aws.CredentialsProvider } func NewAWSCredentials(id, secret, token string, runningOnEc2 bool) (AWSCredentials, error) { var creds aws.Credentials + var credsprovider aws.CredentialsProvider + var ctx context.Context if id != "" && secret != "" { - provider := credentials.NewStaticCredentialsProvider(id, secret, token) - var err error - if creds, err = provider.Retrieve(context.TODO()); err != nil { + cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(id, secret, token))) + if err != nil { return nil, err } - } else { - defaultCfg, err := config.LoadDefaultConfig(context.TODO()) + + creds, err = cfg.Credentials.Retrieve(context.Background()) if err != nil { return nil, err } - if runningOnEc2 { - defaultCfg.HTTPClient = http.NewBuildableClient().WithTimeout(10 * time.Second) - defaultProvider := config.WithCredentialsProvider(defaultCfg.Credentials) - ec2provider := config.WithCredentialsProvider(ec2rolecreds.New(func(o *ec2rolecreds.Options) { - o.Client = imds.NewFromConfig(defaultCfg) - })) + } else if runningOnEc2 { - cfg, err := config.LoadDefaultConfig(context.TODO(), - defaultProvider, - ec2provider, - ) - if err != nil { - log.Fatal(err) - } + ec2Provider := ec2rolecreds.New(func(o *ec2rolecreds.Options) { + o.Client = imds.New(imds.Options{ + HTTPClient: http.NewBuildableClient().WithTimeout(10 * time.Second), + }) + }) - creds, err = cfg.Credentials.Retrieve(context.Background()) - if err != nil { - return nil, err - } + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(ec2Provider), + ) + if err != nil { + return nil, err + } + + creds, err = cfg.Credentials.Retrieve(context.Background()) + if err != nil { + return nil, err } } - return &awsCred{&creds}, nil + return &awsCred{&creds, credsprovider}, nil } func (a *awsCred) Get() (*aws.Credentials, error) { + if a.creds.Expired() { + // Refresh the credentials + if _, err := a.credsprovider.Retrieve(context.TODO()); err != nil { + return nil, err + } + } return a.creds, nil } From 6a1cf46cb0faa8455010bce92100ac7ce4d99d69 Mon Sep 17 00:00:00 2001 From: arivankar-px Date: Sun, 3 Sep 2023 21:21:35 +0530 Subject: [PATCH 16/16] Made changes based on review comments --- aws/aws_kms/aws_kms.go | 14 +------------- aws/aws_secrets_manager/aws_scm.go | 17 ++--------------- aws/credentials/credentials.go | 5 +++++ 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/aws/aws_kms/aws_kms.go b/aws/aws_kms/aws_kms.go index edcd629c..dd38ff4e 100644 --- a/aws/aws_kms/aws_kms.go +++ b/aws/aws_kms/aws_kms.go @@ -9,7 +9,6 @@ import ( "github.com/libopenstorage/secrets/aws/utils" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" "github.com/libopenstorage/secrets" @@ -85,7 +84,7 @@ func New( if err != nil { return nil, fmt.Errorf("Failed to get credentials: %v", err) } - credProv := credentialsToProvider(creds) + credProv, err := asc.GetCredentialsProvider() config := aws.Config{ Credentials: credProv, Region: region, @@ -102,17 +101,6 @@ func New( }, nil } -func credentialsToProvider(creds *aws.Credentials) aws.CredentialsProvider { - return credentials.StaticCredentialsProvider{ - Value: aws.Credentials{ - AccessKeyID: creds.AccessKeyID, - SecretAccessKey: creds.SecretAccessKey, - SessionToken: creds.SessionToken, - Source: creds.Source, - }, - } -} - func (a *awsKmsSecrets) String() string { return Name } diff --git a/aws/aws_secrets_manager/aws_scm.go b/aws/aws_secrets_manager/aws_scm.go index 5f4cb904..f2c6f80c 100644 --- a/aws/aws_secrets_manager/aws_scm.go +++ b/aws/aws_secrets_manager/aws_scm.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" "github.com/libopenstorage/secrets" @@ -64,11 +63,11 @@ func New( if err != nil { return nil, fmt.Errorf("failed to create aws credentials instance: %v", err) } - creds, err := asc.Get() + _, err = asc.Get() if err != nil { return nil, fmt.Errorf("failed to get credentials: %v", err) } - credProv := CredentialsToProvider(creds) + credProv, err := asc.GetCredentialsProvider() config := aws.Config{ Credentials: credProv, Region: region, @@ -77,18 +76,6 @@ func New( return NewFromAWSConfig(config) } -// credentialsToProvider converts a aws.Credential object to a aws.CredentialProvider object -func CredentialsToProvider(creds *aws.Credentials) aws.CredentialsProvider { - return credentials.StaticCredentialsProvider{ - Value: aws.Credentials{ - AccessKeyID: creds.AccessKeyID, - SecretAccessKey: creds.SecretAccessKey, - SessionToken: creds.SessionToken, - Source: creds.Source, - }, - } -} - // NewFromAWSConfig creates new instance of AWSSecretsMgr with provided AWS configuration (aws.Config). func NewFromAWSConfig(config aws.Config) (*AWSSecretsMgr, error) { scm := secretsmanager.NewFromConfig(config) diff --git a/aws/credentials/credentials.go b/aws/credentials/credentials.go index eb0c0095..ddb6719d 100644 --- a/aws/credentials/credentials.go +++ b/aws/credentials/credentials.go @@ -14,6 +14,7 @@ import ( type AWSCredentials interface { Get() (*aws.Credentials, error) + GetCredentialsProvider() (aws.CredentialsProvider, error) } type awsCred struct { @@ -68,3 +69,7 @@ func (a *awsCred) Get() (*aws.Credentials, error) { } return a.creds, nil } + +func (a *awsCred) GetCredentialsProvider() (aws.CredentialsProvider, error) { + return a.credsprovider, nil +}