From cce9915091823a1502057156e435e5a597352025 Mon Sep 17 00:00:00 2001 From: Zoran Rajic Date: Fri, 12 Apr 2024 23:16:06 -0700 Subject: [PATCH] PWX-36873: vault cooldowns * double "permission denied" REST error will put vault client into 5 minutes cooldown for all REST calls * can disable via `VAULT_COOLDOWN_PERIOD:disabled` Signed-off-by: Zoran Rajic --- vault/vault.go | 109 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 17 deletions(-) diff --git a/vault/vault.go b/vault/vault.go index 15916b7c..4f5e76fb 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -1,10 +1,12 @@ package vault import ( + "errors" "fmt" "path" "strings" "sync" + "time" "github.com/hashicorp/vault/api" "github.com/libopenstorage/secrets" @@ -13,15 +15,17 @@ import ( ) const ( - Name = secrets.TypeVault - DefaultBackendPath = "secret/" - VaultBackendPathKey = "VAULT_BACKEND_PATH" - VaultBackendKey = "VAULT_BACKEND" - kvVersionKey = "version" - kvDataKey = "data" - kvMetadataKey = "metadata" - kvVersion1 = "kv" - kvVersion2 = "kv-v2" + Name = secrets.TypeVault + DefaultBackendPath = "secret/" + VaultBackendPathKey = "VAULT_BACKEND_PATH" + VaultBackendKey = "VAULT_BACKEND" + VaultCooldownPeriod = "VAULT_COOLDOWN_PERIOD" + kvVersionKey = "version" + kvDataKey = "data" + kvMetadataKey = "metadata" + kvVersion1 = "kv" + kvVersion2 = "kv-v2" + defaultCooldownPeriod = 5 * time.Minute AuthMethodKubernetes = utils.AuthMethodKubernetes AuthMethod = utils.AuthMethod @@ -52,12 +56,15 @@ type vaultSecrets struct { isKvBackendV2 bool autoAuth bool config map[string]interface{} + cooldown time.Time } // These variables are helpful in testing to stub method call from packages var ( - newVaultClient = api.NewClient - isKvV2 = isKvBackendV2 + newVaultClient = api.NewClient + isKvV2 = isKvBackendV2 + errInCooldown = errors.New("vault client is in cooldown") + confCooldownPeriod time.Duration ) func New( @@ -109,7 +116,7 @@ func New( authMethod = method } - logrus.Infof("Authenticated to Vault with %v\n", authMethod) + logrus.Infof("Will authenticate to Vault via %v", authMethod) backendPath := utils.GetVaultParam(secretConfig, VaultBackendPathKey) if backendPath == "" { @@ -130,6 +137,21 @@ func New( return nil, err } } + + if cd := utils.GetVaultParam(secretConfig, VaultCooldownPeriod); cd != "" { + confCooldownPeriod, err = time.ParseDuration(cd) + if err == nil && confCooldownPeriod > time.Minute { + logrus.Infof("cooldown period is set to %s via %s=%s", confCooldownPeriod, VaultCooldownPeriod, cd) + } else { + // let's turn OFF the cooldown, if one passed invalid value (or "disabled") + logrus.WithError(err).Warnf("cooldown period is invalid (%s=%s) -- cooldowns turned OFF", VaultCooldownPeriod, cd) + confCooldownPeriod = 0 + } + } else { + confCooldownPeriod = defaultCooldownPeriod + logrus.Infof("cooldown period is set to %s", confCooldownPeriod) + } + return &vaultSecrets{ endpoint: config.Address, namespace: namespace, @@ -261,6 +283,9 @@ func (v *vaultSecrets) ListSecrets() ([]string, error) { } func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) { + if v.isInCooldown() { + return nil, errInCooldown + } if v.autoAuth { v.lockClientToken.Lock() defer v.lockClientToken.Unlock() @@ -272,7 +297,7 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) { secretValue, err := v.lockedRead(path.Path()) if v.isTokenExpired(err) { - if err = v.renewToken(path.Namespace()); err != nil { + if err = v.renewTokenWithCooldown(path.Namespace()); err != nil { return nil, fmt.Errorf("failed to renew token: %s", err) } return v.lockedRead(path.Path()) @@ -281,6 +306,9 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) { } func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Secret, error) { + if v.isInCooldown() { + return nil, errInCooldown + } if v.autoAuth { v.lockClientToken.Lock() defer v.lockClientToken.Unlock() @@ -292,7 +320,7 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se secretValue, err := v.lockedWrite(path.Path(), data) if v.isTokenExpired(err) { - if err = v.renewToken(path.Namespace()); err != nil { + if err = v.renewTokenWithCooldown(path.Namespace()); err != nil { return nil, fmt.Errorf("failed to renew token: %s", err) } return v.lockedWrite(path.Path(), data) @@ -301,6 +329,9 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se } func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) { + if v.isInCooldown() { + return nil, errInCooldown + } if v.autoAuth { v.lockClientToken.Lock() defer v.lockClientToken.Unlock() @@ -312,7 +343,7 @@ func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) { secretValue, err := v.lockedDelete(path.Path()) if v.isTokenExpired(err) { - if err = v.renewToken(path.Namespace()); err != nil { + if err = v.renewTokenWithCooldown(path.Namespace()); err != nil { return nil, fmt.Errorf("failed to renew token: %s", err) } return v.lockedDelete(path.Path()) @@ -359,6 +390,50 @@ func (v *vaultSecrets) renewToken(namespace string) error { return nil } +func (v *vaultSecrets) renewTokenWithCooldown(namespace string) error { + if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately + return v.renewToken(namespace) + } else if v.isInCooldown() { + return errInCooldown + } + + err := v.renewToken(namespace) + if v.isTokenExpired(err) { + v.setCooldown(confCooldownPeriod) + } else if err == nil { + v.setCooldown(0) // clear cooldown + } + return err +} + +func (v *vaultSecrets) isInCooldown() bool { + if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately + return false + } + v.mu.RLock() + defer v.mu.RUnlock() + if v.cooldown.IsZero() { + return false + } + return time.Now().Before(v.cooldown) +} + +func (v *vaultSecrets) setCooldown(dur time.Duration) { + if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately + return + } + v.mu.Lock() + defer v.mu.Unlock() + if dur > 0 { + v.cooldown = time.Now().Add(dur) + logrus.WithField("nextRetryAt", v.cooldown.Round(100*time.Millisecond)). + Warnf("putting vault client in cooldown for %s", confCooldownPeriod) + } else { + logrus.Debug("clearing vault client cooldown") + v.cooldown = time.Time{} + } +} + func (v *vaultSecrets) isTokenExpired(err error) bool { return err != nil && v.autoAuth && strings.Contains(err.Error(), "permission denied") } @@ -373,7 +448,7 @@ func (v *vaultSecrets) setNamespaceToken(namespace string) error { return nil } - return v.renewToken(namespace) + return v.renewTokenWithCooldown(namespace) } func isKvBackendV2(client *api.Client, backendPath string) (bool, error) { @@ -393,7 +468,7 @@ func isKvBackendV2(client *api.Client, backendPath string) (bool, error) { } } - return false, fmt.Errorf("Secrets engine with mount path '%s' not found", + return false, fmt.Errorf("secrets engine with mount path '%s' not found", backendPath) }