PWX-36873: vault cooldowns

* double "permission denied" REST error will put vault client into 5 minutes cooldown for all REST calls * can disable via `VAULT_COOLDOWN_PERIOD:disabled` Signed-off-by: Zoran Rajic <[email protected]>
libopenstorage · Apr 13, 2024 · cce9915 · cce9915
1 parent 5f4b25c
commit cce9915
Showing 1 changed file with 92 additions and 17 deletions.
diff --git a/vault/vault.go b/vault/vault.go
@@ -1,10 +1,12 @@
 package vault
 
 import (
+	"errors"
 	"fmt"
 	"path"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/hashicorp/vault/api"
 	"github.com/libopenstorage/secrets"
@@ -13,15 +15,17 @@ import (
 )
 
 const (
-	Name                = secrets.TypeVault
-	DefaultBackendPath  = "secret/"
-	VaultBackendPathKey = "VAULT_BACKEND_PATH"
-	VaultBackendKey     = "VAULT_BACKEND"
-	kvVersionKey        = "version"
-	kvDataKey           = "data"
-	kvMetadataKey       = "metadata"
-	kvVersion1          = "kv"
-	kvVersion2          = "kv-v2"
+	Name                  = secrets.TypeVault
+	DefaultBackendPath    = "secret/"
+	VaultBackendPathKey   = "VAULT_BACKEND_PATH"
+	VaultBackendKey       = "VAULT_BACKEND"
+	VaultCooldownPeriod   = "VAULT_COOLDOWN_PERIOD"
+	kvVersionKey          = "version"
+	kvDataKey             = "data"
+	kvMetadataKey         = "metadata"
+	kvVersion1            = "kv"
+	kvVersion2            = "kv-v2"
+	defaultCooldownPeriod = 5 * time.Minute
 
 	AuthMethodKubernetes    = utils.AuthMethodKubernetes
 	AuthMethod              = utils.AuthMethod
@@ -52,12 +56,15 @@ type vaultSecrets struct {
 	isKvBackendV2 bool
 	autoAuth      bool
 	config        map[string]interface{}
+	cooldown      time.Time
 }
 
 // These variables are helpful in testing to stub method call from packages
 var (
-	newVaultClient = api.NewClient
-	isKvV2         = isKvBackendV2
+	newVaultClient     = api.NewClient
+	isKvV2             = isKvBackendV2
+	errInCooldown      = errors.New("vault client is in cooldown")
+	confCooldownPeriod time.Duration
 )
 
 func New(
@@ -109,7 +116,7 @@ func New(
 		authMethod = method
 	}
 
-	logrus.Infof("Authenticated to Vault with %v\n", authMethod)
+	logrus.Infof("Will authenticate to Vault via %v", authMethod)
 
 	backendPath := utils.GetVaultParam(secretConfig, VaultBackendPathKey)
 	if backendPath == "" {
@@ -130,6 +137,21 @@ func New(
 			return nil, err
 		}
 	}
+
+	if cd := utils.GetVaultParam(secretConfig, VaultCooldownPeriod); cd != "" {
+		confCooldownPeriod, err = time.ParseDuration(cd)
+		if err == nil && confCooldownPeriod > time.Minute {
+			logrus.Infof("cooldown period is set to %s via %s=%s", confCooldownPeriod, VaultCooldownPeriod, cd)
+		} else {
+			// let's turn OFF the cooldown, if one passed invalid value (or "disabled")
+			logrus.WithError(err).Warnf("cooldown period is invalid (%s=%s) -- cooldowns turned OFF", VaultCooldownPeriod, cd)
+			confCooldownPeriod = 0
+		}
+	} else {
+		confCooldownPeriod = defaultCooldownPeriod
+		logrus.Infof("cooldown period is set to %s", confCooldownPeriod)
+	}
+
 	return &vaultSecrets{
 		endpoint:         config.Address,
 		namespace:        namespace,
@@ -261,6 +283,9 @@ func (v *vaultSecrets) ListSecrets() ([]string, error) {
 }
 
 func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
+	if v.isInCooldown() {
+		return nil, errInCooldown
+	}
 	if v.autoAuth {
 		v.lockClientToken.Lock()
 		defer v.lockClientToken.Unlock()
@@ -272,7 +297,7 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
 
 	secretValue, err := v.lockedRead(path.Path())
 	if v.isTokenExpired(err) {
-		if err = v.renewToken(path.Namespace()); err != nil {
+		if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
 			return nil, fmt.Errorf("failed to renew token: %s", err)
 		}
 		return v.lockedRead(path.Path())
@@ -281,6 +306,9 @@ func (v *vaultSecrets) read(path keyPath) (*api.Secret, error) {
 }
 
 func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Secret, error) {
+	if v.isInCooldown() {
+		return nil, errInCooldown
+	}
 	if v.autoAuth {
 		v.lockClientToken.Lock()
 		defer v.lockClientToken.Unlock()
@@ -292,7 +320,7 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se
 
 	secretValue, err := v.lockedWrite(path.Path(), data)
 	if v.isTokenExpired(err) {
-		if err = v.renewToken(path.Namespace()); err != nil {
+		if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
 			return nil, fmt.Errorf("failed to renew token: %s", err)
 		}
 		return v.lockedWrite(path.Path(), data)
@@ -301,6 +329,9 @@ func (v *vaultSecrets) write(path keyPath, data map[string]interface{}) (*api.Se
 }
 
 func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {
+	if v.isInCooldown() {
+		return nil, errInCooldown
+	}
 	if v.autoAuth {
 		v.lockClientToken.Lock()
 		defer v.lockClientToken.Unlock()
@@ -312,7 +343,7 @@ func (v *vaultSecrets) delete(path keyPath) (*api.Secret, error) {
 
 	secretValue, err := v.lockedDelete(path.Path())
 	if v.isTokenExpired(err) {
-		if err = v.renewToken(path.Namespace()); err != nil {
+		if err = v.renewTokenWithCooldown(path.Namespace()); err != nil {
 			return nil, fmt.Errorf("failed to renew token: %s", err)
 		}
 		return v.lockedDelete(path.Path())
@@ -359,6 +390,50 @@ func (v *vaultSecrets) renewToken(namespace string) error {
 	return nil
 }
 
+func (v *vaultSecrets) renewTokenWithCooldown(namespace string) error {
+	if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
+		return v.renewToken(namespace)
+	} else if v.isInCooldown() {
+		return errInCooldown
+	}
+
+	err := v.renewToken(namespace)
+	if v.isTokenExpired(err) {
+		v.setCooldown(confCooldownPeriod)
+	} else if err == nil {
+		v.setCooldown(0) // clear cooldown
+	}
+	return err
+}
+
+func (v *vaultSecrets) isInCooldown() bool {
+	if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
+		return false
+	}
+	v.mu.RLock()
+	defer v.mu.RUnlock()
+	if v.cooldown.IsZero() {
+		return false
+	}
+	return time.Now().Before(v.cooldown)
+}
+
+func (v *vaultSecrets) setCooldown(dur time.Duration) {
+	if confCooldownPeriod <= 0 { // cooldown is disabled, return immediately
+		return
+	}
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	if dur > 0 {
+		v.cooldown = time.Now().Add(dur)
+		logrus.WithField("nextRetryAt", v.cooldown.Round(100*time.Millisecond)).
+			Warnf("putting vault client in cooldown for %s", confCooldownPeriod)
+	} else {
+		logrus.Debug("clearing vault client cooldown")
+		v.cooldown = time.Time{}
+	}
+}
+
 func (v *vaultSecrets) isTokenExpired(err error) bool {
 	return err != nil && v.autoAuth && strings.Contains(err.Error(), "permission denied")
 }
@@ -373,7 +448,7 @@ func (v *vaultSecrets) setNamespaceToken(namespace string) error {
 		return nil
 	}
 
-	return v.renewToken(namespace)
+	return v.renewTokenWithCooldown(namespace)
 }
 
 func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
@@ -393,7 +468,7 @@ func isKvBackendV2(client *api.Client, backendPath string) (bool, error) {
 		}
 	}
 
-	return false, fmt.Errorf("Secrets engine with mount path '%s' not found",
+	return false, fmt.Errorf("secrets engine with mount path '%s' not found",
 		backendPath)
 }