Leader detection

README.md

@@ -142,8 +142,10 @@ The Agent monitors the configuration-file for changes and reloads the configurat
 
 ```
 vault:
-  # Url of the (leading) vault-server
-  url: https://vault-server:8200
+  nodes:
+    urls:
+      # Url of the (leading) vault-server
+      - https://vault-server:8200
   auth:
     # configures kubernetes auth
     kubernetes:
@@ -177,14 +179,14 @@ to that storage will fail (gracefully)!**
 
 ```
 vault:
-  url: <http(s)-url to vault-server>
+  url: <http(s)-url to vault-cluster leader>
   insecure: <true|false>
   timeout: <duration>
 ```
 
 | Key                             | Type                                                   | Required/*Default*       | Description                                                                                                          |
 | ------------------------------- | ------------------------------------------------------ | ------------------------ | -------------------------------------------------------------------------------------------------------------------- |
-| <a id="cnf-vault-url"></a>`url` | URL                                                    | *https://127.0.0.1:8200* | specifies the url of the vault-server                                                                                |
+| <a id="cnf-vault-url"></a>`url` | URL                                                    | *https://127.0.0.1:8200* | specifies the url of the vault-server (*DEPRECATED, use nodes instead*)                                                                               |
 | `insecure`                      | Boolean                                                | *false*                  | specifies whether insecure https connections are allowed or not. Set to `true` when you use self-signed certificates |
 | `timeout`                       | [Duration](https://golang.org/pkg/time/#ParseDuration) | *60s*                    | timeout for the vault-http-client; increase for large raft databases (and increase `snapshots.timeout` accordingly!) |
 
@@ -193,6 +195,24 @@ elected leader!** When running Vault on Kubernetes installed by
 the [default helm-chart](https://developer.hashicorp.com/vault/docs/platform/k8s/helm), this should be
 `http(s)://vault-active.<vault-namespace>.svc.cluster.local:<vault-server service-port>`.|
 
+### Vault Nodes configuration
+While it is still recommended to have a single url which always points to the cluster leader, you may provide a list of urls to all known nodes that are reachable from the agent and let it figure out, which one is the leader. 
+
+```
+vault:
+  nodes:
+    urls:
+      -  <http(s)-urls to vault-cluster nodes>
+      - ...
+    autoDetectLeader: true
+```
+
+| Key                             | Type                                                   | Required/*Default*       | Description                                                                                                          |
+| ------------------------------- | ------------------------------------------------------ | ------------------------ | -------------------------------------------------------------------------------------------------------------------- |
+| <a id="cnf-vault-url"></a>`nodes.urls` | List of URL                                                    | **required** | specifies at least one url to a vault-server                                                                                |
+| `nodes.autoDetectLeader`               | Boolean                                          | *false*                  | if true the agent will ask the nodes for the url to the leader. Otherwise it will try the given urls until it finds the leader node |
+
+
 ### Vault authentication
 
 To allow Vault Raft Snapshot Agent to take snapshots, you must add a policy that allows read-access to the

internal/agent/snapshot-agent_test.go

@@ -3,13 +3,14 @@ package agent
 import (
 	"context"
 	"errors"
-	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/storage"
-	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/vault"
-	"github.com/hashicorp/vault/api"
 	"io"
 	"testing"
 	"time"
 
+	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/storage"
+	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/vault"
+	"github.com/hashicorp/vault/api"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -23,7 +24,8 @@ func TestTakeSnapshotUploadsSnapshot(t *testing.T) {
 		Frequency: time.Millisecond,
 	}
 
-	factory := &storageControllerFactoryStub{nextSnapshot: time.Now().Add(time.Millisecond * 250)}
+	expectedNextSnapshot := time.Now().Add(time.Millisecond * 250)
+	factory := &storageControllerFactoryStub{nextSnapshot: expectedNextSnapshot}
 
 	manager := &storage.Manager{}
 	manager.AddStorageFactory(factory)
@@ -41,7 +43,7 @@ func TestTakeSnapshotUploadsSnapshot(t *testing.T) {
 	assert.Equal(t, clientVaultAPI.snapshotData, factory.uploadData)
 	assert.Equal(t, defaults, factory.defaults)
 	assert.WithinRange(t, factory.snapshotTimestamp, start, start.Add(50*time.Millisecond))
-	assert.GreaterOrEqual(t, time.Now(), factory.nextSnapshot)
+	assert.Equal(t, expectedNextSnapshot, factory.nextSnapshot)
 }
 
 func TestTakeSnapshotLocksTakeSnapshot(t *testing.T) {
@@ -259,7 +261,7 @@ func TestUpdateReschedulesSnapshots(t *testing.T) {
 }
 
 func newClient(api *clientVaultAPIStub) *vault.VaultClient {
-	return vault.NewClient(api, clientVaultAPIAuthStub{}, time.Time{})
+	return vault.NewClient(api, []string{"http://node"}, false, clientVaultAPIAuthStub{})
 }
 
 type clientVaultAPIStub struct {
@@ -270,11 +272,14 @@ type clientVaultAPIStub struct {
 	snapshotData    string
 }
 
-func (stub *clientVaultAPIStub) Address() string {
-	return "test"
+func (stub *clientVaultAPIStub) Connect(node string) (*api.Client, error) {
+	config := api.DefaultConfig()
+	config.Address = node
+
+	return api.NewClient(config)
 }
 
-func (stub *clientVaultAPIStub) TakeSnapshot(ctx context.Context, writer io.Writer) error {
+func (stub *clientVaultAPIStub) TakeSnapshot(ctx context.Context, _ *api.Client, writer io.Writer) error {
 	stub.tookSnapshot = true
 	if stub.snapshotFails {
 		return errors.New("TakeSnapshot failed")
@@ -294,19 +299,14 @@ func (stub *clientVaultAPIStub) TakeSnapshot(ctx context.Context, writer io.Writ
 	return nil
 }
 
-func (stub *clientVaultAPIStub) IsLeader() (bool, error) {
-	return stub.leader, nil
-}
-
-func (stub *clientVaultAPIStub) RefreshAuth(ctx context.Context, auth api.AuthMethod) (time.Duration, error) {
-	_, err := auth.Login(ctx, nil)
-	return 0, err
+func (stub *clientVaultAPIStub) GetLeader(context.Context, *api.Client) (bool, string) {
+	return stub.leader, ""
 }
 
 type clientVaultAPIAuthStub struct{}
 
-func (stub clientVaultAPIAuthStub) Login(_ context.Context, _ *api.Client) (*api.Secret, error) {
-	return nil, nil
+func (stub clientVaultAPIAuthStub) Refresh(context.Context, *api.Client, bool) error {
+	return nil
 }
 
 type storageControllerFactoryStub struct {

internal/agent/vault/auth/auth.go

@@ -3,57 +3,74 @@ package auth
 import (
 	"context"
 	"fmt"
+	"time"
+
 	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/agent/logging"
 	"github.com/hashicorp/vault/api"
 )
 
-type VaultAuthConfig struct {
-	AppRole    *AppRoleAuthConfig
-	AWS        *AWSAuthConfig
-	Azure      *AzureAuthConfig
-	GCP        *GCPAuthConfig
-	Kubernetes *KubernetesAuthConfig
-	LDAP       *LDAPAuthConfig
-	UserPass   *UserPassAuthConfig
-	Token      *Token
+type VaultAuth interface {
+	Refresh(context.Context, *api.Client, bool) error
 }
 
 type vaultAuthMethodFactory interface {
 	createAuthMethod() (api.AuthMethod, error)
 }
 
-type vaultAuthMethodImpl struct {
-	methodFactory vaultAuthMethodFactory
+type vaultAuthImpl struct {
+	factory vaultAuthMethodFactory
+	expires time.Time
 }
 
-func CreateVaultAuth(config VaultAuthConfig) (api.AuthMethod, error) {
+func CreateVaultAuth(config VaultAuthConfig) (VaultAuth, error) {
 	if config.AppRole != nil {
-		return vaultAuthMethodImpl{config.AppRole}, nil
+		return &vaultAuthImpl{factory: config.AppRole}, nil
 	} else if config.AWS != nil {
-		return vaultAuthMethodImpl{config.AWS}, nil
+		return &vaultAuthImpl{factory: config.AWS}, nil
 	} else if config.Azure != nil {
-		return vaultAuthMethodImpl{config.Azure}, nil
+		return &vaultAuthImpl{factory: config.Azure}, nil
 	} else if config.GCP != nil {
-		return vaultAuthMethodImpl{config.GCP}, nil
+		return &vaultAuthImpl{factory: config.GCP}, nil
 	} else if config.Kubernetes != nil {
-		return vaultAuthMethodImpl{config.Kubernetes}, nil
+		return &vaultAuthImpl{factory: config.Kubernetes}, nil
 	} else if config.LDAP != nil {
-		return vaultAuthMethodImpl{config.LDAP}, nil
+		return &vaultAuthImpl{factory: config.LDAP}, nil
 	} else if config.UserPass != nil {
-		return vaultAuthMethodImpl{config.UserPass}, nil
+		return &vaultAuthImpl{factory: config.UserPass}, nil
 	} else if config.Token != nil {
-		return vaultAuthMethodImpl{config.Token}, nil
+		return &vaultAuthImpl{factory: config.Token}, nil
 	} else {
 		return nil, fmt.Errorf("unknown authenticatin method")
 	}
 }
 
-func (am vaultAuthMethodImpl) Login(ctx context.Context, client *api.Client) (*api.Secret, error) {
-	method, err := am.methodFactory.createAuthMethod()
+func (auth *vaultAuthImpl) Refresh(ctx context.Context, client *api.Client, force bool) error {
+	if !force && auth.expires.After(time.Now()) {
+		return nil
+	}
+
+	method, err := auth.factory.createAuthMethod()
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	logging.Debug("Logging into vault", "method", fmt.Sprintf("%T", method))
-	return client.Auth().Login(ctx, method)
+	authSecret, err := client.Auth().Login(ctx, method)
+	if err != nil {
+		return err
+	}
+
+	tokenTTL, err := authSecret.TokenTTL()
+	if err != nil {
+		return err
+	}
+
+	tokenPolicies, err := authSecret.TokenPolicies()
+	if err != nil {
+		return err
+	}
+
+	auth.expires = time.Now().Add(tokenTTL / 2)
+	logging.Debug("Successfully logged in ", "policies", tokenPolicies, "ttl", tokenTTL, "expires", auth.expires)
+	return nil
 }

internal/agent/vault/auth/auth_test.go

@@ -3,50 +3,100 @@ package auth
 import (
 	"context"
 	"errors"
+	"testing"
+	"time"
+
 	"github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
-func TestVaultAuthMethod_Login_FailsIfMethodFactoryFails(t *testing.T) {
+func TestVaultAuth_Refresh_FailsIfMethodFactoryFails(t *testing.T) {
 	expectedErr := errors.New("create failed")
-	auth := vaultAuthMethodImpl{
-		authMethodFactoryStub{createErr: expectedErr},
+	auth := vaultAuthImpl{
+		factory: authMethodFactoryStub{createErr: expectedErr},
+	}
+
+	err := auth.Refresh(context.Background(), nil, true)
+	assert.ErrorIs(t, err, expectedErr)
+	assert.Equal(t, time.Time{}, auth.expires)
+}
+
+func TestVaultAuth_Refresh_FailsIfAuthMethodRefreshFails(t *testing.T) {
+	expectedErr := errors.New("login failed")
+	auth := vaultAuthImpl{
+		factory: authMethodFactoryStub{
+			method: authMethodStub{loginError: expectedErr},
+		},
 	}
 
-	_, err := auth.Login(context.Background(), nil)
+	err := auth.Refresh(context.Background(), nil, true)
 	assert.ErrorIs(t, err, expectedErr)
+	assert.Equal(t, time.Time{}, auth.expires)
 }
 
-func TestVaultAuthMethod_Login_FailsIfAuthMethodLoginFails(t *testing.T) {
+func TestVaultAuth_Refresh_SkipsLoginUntilExpired(t *testing.T) {
+	expires := time.Now().Add(time.Duration(1) * time.Second)
 	expectedErr := errors.New("login failed")
-	auth := vaultAuthMethodImpl{
-		authMethodFactoryStub{
+	auth := vaultAuthImpl{
+		factory: authMethodFactoryStub{
 			method: authMethodStub{loginError: expectedErr},
 		},
+		expires: expires,
 	}
 
-	_, err := auth.Login(context.Background(), nil)
+	err := auth.Refresh(context.Background(), nil, false)
+	assert.NoError(t, err, "Refresh failed unexpectedly")
+	assert.Equal(t, expires, auth.expires)
+
+	time.Sleep(time.Second * 2)
+
+	err = auth.Refresh(context.Background(), nil, false)
 	assert.ErrorIs(t, err, expectedErr)
 }
 
-func TestVaultAuthMethod_Login_ReturnsLeaseDuration(t *testing.T) {
+func TestVaultAuth_Expires_AfterTokenTTL(t *testing.T) {
+	expectedTTL := 60
+	expectedExpires := time.Now().Add((time.Duration(expectedTTL) * time.Second) / 2)
 	expectedSecret := &api.Secret{
 		Auth: &api.SecretAuth{
-			ClientToken: "test",
+			ClientToken:   "test",
+			LeaseDuration: expectedTTL,
 		},
 	}
 
-	auth := vaultAuthMethodImpl{
-		authMethodFactoryStub{
+	auth := vaultAuthImpl{
+		factory: authMethodFactoryStub{
 			method: authMethodStub{secret: expectedSecret},
 		},
 	}
 
-	authSecret, err := auth.Login(context.Background(), &api.Client{})
+	err := auth.Refresh(context.Background(), &api.Client{}, false)
+
+	assert.NoError(t, err, "Refresh failed unexpectedly")
+	assert.WithinRange(t, auth.expires, expectedExpires, expectedExpires.Add(50*time.Millisecond))
+}
+
+func TestVaultAuth_Refresh_IgnoresExpiresIfForced(t *testing.T) {
+	expires := time.Now().Add(time.Duration(60) * time.Second)
+	expectedTTL := 30
+	expectedExpires := time.Now().Add((time.Duration(expectedTTL) * time.Second) / 2)
+	expectedSecret := &api.Secret{
+		Auth: &api.SecretAuth{
+			ClientToken:   "test",
+			LeaseDuration: expectedTTL,
+		},
+	}
+
+	auth := vaultAuthImpl{
+		factory: authMethodFactoryStub{
+			method: authMethodStub{secret: expectedSecret},
+		},
+		expires: expires,
+	}
 
-	assert.NoError(t, err, "Login failed unexpectedly")
-	assert.Equal(t, expectedSecret, authSecret)
+	err := auth.Refresh(context.Background(), &api.Client{}, true)
+	assert.NoError(t, err, "Refresh failed unexpectedly")
+	assert.WithinRange(t, auth.expires, expectedExpires, expectedExpires.Add(50*time.Millisecond))
 }
 
 type authMethodFactoryStub struct {

internal/agent/vault/auth/config.go

@@ -0,0 +1,13 @@
+package auth
+
+
+type VaultAuthConfig struct {
+	AppRole    *AppRoleAuthConfig
+	AWS        *AWSAuthConfig
+	Azure      *AzureAuthConfig
+	GCP        *GCPAuthConfig
+	Kubernetes *KubernetesAuthConfig
+	LDAP       *LDAPAuthConfig
+	UserPass   *UserPassAuthConfig
+	Token      *Token
+}