Uploader specific frequency and retain (#14)

* refactor storages & storage management to allow individual frequency configuration etc. for storages (as proposed in Lucretius#11)
Argelbargel · Sep 22, 2023 · 7158dc4 · 7158dc4
1 parent f5919a4
commit 7158dc4
Show file tree

Hide file tree

Showing 29 changed files with 2,150 additions and 1,656 deletions.
diff --git a/README.md b/README.md
@@ -483,7 +483,27 @@ to `timestampFormat` and `nameSuffix`, e.g. the defaults would generate
 `raft-snapshot-2023-09-01T15-30-00Z+0200.snap` for a snapshot taken at 15:30:00 on 09/01/2023 when the timezone is
 CEST (GMT + 2h).
 
-### Uploader configuration
+The options below snapshots can be overridden for a specific storage:
+```
+snapshots:
+  frequency: 1h
+  retain: 24
+  storages:
+    local:
+      path: /snapshots
+    aws:
+      frequency: 24h
+      retain: 365
+      timestampFormat: 2006-01-02
+      #...
+```
+In this example the agent would take and store a snapshot to the local-storage every hour, retaining 24 snapshots and
+store a daily snapshot on aws remote storage, retaining the last 365 snapshots with a appropriate shorter timestamp.
+
+*Note: as the agent uses the default frequency in case of failures, you should always configure the shorter frequency in
+the defaults and specify longer frequencies for specific storages if required!*
+
+### Storage configuration
 
 Note that if you specify more than one storage option, *all* specified storages will be written to. For example,
 specifying `local` and `aws` will write to both locations.
@@ -495,9 +515,10 @@ it is currently not possible to e.g. upload to multiple aws regions by specifyin
 
 ##### Minimal Configuration
 ```
-uploaders:
-  aws:
-    bucket: <bucket>
+snapshots:
+  storage
+    aws:
+      bucket: <bucket>
 ```
 
 ##### Configuration Options
@@ -513,13 +534,16 @@ uploaders:
 | `useServerSideEncryption` | Boolean                                          | *false*                       | Set to true to turn on AWS' AES256 encryption. Support for AWS KMS keys is not currently supported                |
 | `forcePathStyle`          | Boolean                                          | *false*                       | needed if your S3 Compatible storage supports only path-style, or you would like to use S3's FIPS Endpoint        |
 
+Any option common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration.
+
 #### Azure Storage
 
 ##### Minimal Configuration
 ```
-uploaders:
-  azure:
-    container: <container>
+snapshots:
+  storage:
+    azure:
+      container: <container>
 ```
 
 ##### Configuration Options
@@ -530,38 +554,47 @@ uploaders:
 | `accountKey`  | [Secret](#secrets-and-external-property-sources) | *env://AZURE_STORAGE_KEY*     | the account key of the storage account; **must resolve to non-empty value**  |
 | `cloudDomain` | String                                           | *blob.core.windows.net*       | domain of the cloud-service to use                                           |
 
+Any option common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration.
+
 #### Google Cloud Storage
 ##### Minimal Configuration
 ```
-uploaders:
-  gcp:
-    bucket: <bucket>
+snapshots:
+  storage:
+    gcp:
+      bucket: <bucket>
 ```
 
 ##### Configuration Options
 | Key      | Type   | Required/*Default* | Description                                                                               |
 | -------- | ------ | ------------------ | ----------------------------------------------------------------------------------------- |
 | `bucket` | String | **required**       | the Google Storage Bucket to write to. Auth is expected to be default machine credentials |
 
+Any option common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration.
+
 #### Local Storage
 ##### Minimal Configuration
 ```
-uploaders:
-  local:
-    path: <path>
+snapshots:
+  storages:
+    local:
+      path: <path>
 ```
 ##### Configuration Options
 | Key    | Type   | Required/*Default* | Description                                                                                                     |
 | ------ | ------ | ------------------ | --------------------------------------------------------------------------------------------------------------- |
 | `path` | String | **required**       | fully qualified path, not including file name, for where the snapshot should be written. i.e. `/raft/snapshots` |
 
+Any option common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration.
+
 #### Openstack Swift Storage
 ##### Minimal Configuration
 ```
-uploaders:
-  swift:
-    container: <container>
-    authUrl: <auth-url>
+snapshots:
+  storages:
+    swift:
+      container: <container>
+      authUrl: <auth-url>
 ```
 
 | Key         | Type                                                   | Required/*Default*     | Description                                                               |
@@ -575,6 +608,8 @@ uploaders:
 | `tenantId`  | String                                                 |                        | optional id of the tenant                                                 |
 | `timeout`   | [Duration](https://golang.org/pkg/time/#ParseDuration) | *60s*                  | timeout for snapshot-uploads                                              |
 
+Any option common [snapshot configuration option](#snapshot-configuration) overrides the global snapshot-configuration.
+
 ## License
 
 - Source code is licensed under MIT

diff --git a/cmd/vault-raft-snapshot-agent/main.go b/cmd/vault-raft-snapshot-agent/main.go
@@ -53,7 +53,7 @@ import (
 var Version = "development"
 var Platform = "linux/amd64"
 
-var snapshotterOptions = agent.SnapshotterOptions{
+var agentOptions = agent.SnapshotAgentOptions{
 	ConfigFileName:        "snapshots",
 	ConfigFileSearchPaths: []string{"/etc/vault.d/", "."},
 	EnvPrefix:             "VRSA",
@@ -66,6 +66,36 @@ const (
 	optionLogLevel  = "log-level"
 )
 
+var cliFlags = []cli.Flag{
+	&cli.PathFlag{
+		Name:    optionConfig,
+		Aliases: []string{"c"},
+		Usage:   fmt.Sprintf("load configuration from `FILE`; if not specified, searches for %s.[json|toml|yaml] in /etc/vault.d or the current working directory", agentOptions.ConfigFileName),
+		EnvVars: []string{agentOptions.EnvPrefix + "_CONFIG_FILE"},
+	},
+	&cli.StringFlag{
+		Name:    optionLogFormat,
+		Aliases: []string{"f"},
+		Usage:   "format for log-output; possible values are 'default', 'text', 'json'",
+		EnvVars: []string{agentOptions.EnvPrefix + "_LOG_FORMAT"},
+		Value:   logging.FormatDefault,
+	},
+	&cli.StringFlag{
+		Name:    optionLogOutput,
+		Aliases: []string{"o"},
+		Usage:   "output-target for logs; possible values are 'stderr', 'stdout' or <path-to-logfile>",
+		EnvVars: []string{agentOptions.EnvPrefix + "_LOG_OUTPUT"},
+		Value:   logging.OutputStderr,
+	},
+	&cli.StringFlag{
+		Name:    optionLogLevel,
+		Aliases: []string{"l"},
+		Usage:   "log-level for logs; possible values are 'debug', 'info', 'warn' or 'error'",
+		EnvVars: []string{agentOptions.EnvPrefix + "_LOG_LEVEL"},
+		Value:   logging.LevelInfo,
+	},
+}
+
 type quietBoolFlag struct {
 	cli.BoolFlag
 }
@@ -95,41 +125,15 @@ func main() {
 		Name:        "vault-raft-snapshot-agent",
 		Version:     Version,
 		Description: "takes periodic snapshot of vault's raft-db",
-		Flags: []cli.Flag{
-			&cli.PathFlag{
-				Name:    optionConfig,
-				Aliases: []string{"c"},
-				Usage:   fmt.Sprintf("load configuration from `FILE`; if not specified, searches for %s.[json|toml|yaml] in /etc/vault.d or the current working directory", snapshotterOptions.ConfigFileName),
-				EnvVars: []string{snapshotterOptions.EnvPrefix + "_CONFIG_FILE"},
-			},
-			&cli.StringFlag{
-				Name:    optionLogFormat,
-				Aliases: []string{"f"},
-				Usage:   "format for log-output; possible values are 'default', 'text', 'json'",
-				EnvVars: []string{snapshotterOptions.EnvPrefix + "_LOG_FORMAT"},
-				Value:   logging.FormatDefault,
-			},
-			&cli.StringFlag{
-				Name:    optionLogOutput,
-				Aliases: []string{"o"},
-				Usage:   "output-target for logs; possible values are 'stderr', 'stdout' or <path-to-logfile>",
-				EnvVars: []string{snapshotterOptions.EnvPrefix + "_LOG_OUTPUT"},
-				Value:   logging.OutputStderr,
-			},
-			&cli.StringFlag{
-				Name:    optionLogLevel,
-				Aliases: []string{"l"},
-				Usage:   "log-level for logs; possible values are 'debug', 'info', 'warn' or 'error'",
-				EnvVars: []string{snapshotterOptions.EnvPrefix + "_LOG_LEVEL"},
-				Value:   logging.LevelInfo,
-			},
-		},
+		Flags:       cliFlags,
 		Action: func(ctx *cli.Context) error {
 			err := logging.Configure(ctx.String(optionLogOutput), ctx.String(optionLogFormat), ctx.String(optionLogLevel))
 			if err != nil {
 				log.Fatalf("could not configure logging: %s", err)
 			}
-			return startSnapshotter(ctx.Path(optionConfig))
+
+			agentOptions.ConfigFilePath = ctx.Path(optionConfig)
+			return run()
 		},
 	}
 	app.CustomAppHelpTemplate = `Usage: {{.HelpName}} [options]
@@ -144,13 +148,7 @@ Options:
 	}
 }
 
-func startSnapshotter(configFile cli.Path) error {
-	snapshotterOptions.ConfigFilePath = configFile
-	snapshotter, err := agent.CreateSnapshotter(snapshotterOptions)
-	if err != nil {
-		return err
-	}
-
+func run() error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -161,15 +159,19 @@ func startSnapshotter(configFile cli.Path) error {
 		cancel()
 	}()
 
-	runSnapshotter(ctx, snapshotter)
-	return nil
+	return runAgent(ctx)
 }
 
-func runSnapshotter(ctx context.Context, snapshotter *agent.Snapshotter) {
+func runAgent(ctx context.Context) error {
+	snapshotAgent, err := agent.CreateSnapshotAgent(ctx, agentOptions)
+	if err != nil {
+		return err
+	}
+
 	for {
-		timeout, _ := snapshotter.TakeSnapshot(ctx)
+		nextSnapshotTimer := snapshotAgent.TakeSnapshot(ctx)
 		select {
-		case <-timeout.C:
+		case <-nextSnapshotTimer.C:
 			continue
 		case <-ctx.Done():
 			os.Exit(0)

diff --git a/internal/agent/config/config.go b/internal/agent/config/config.go
@@ -10,7 +10,7 @@ type Parser[T Configuration] struct {
 }
 
 type Configuration interface {
-	HasUploaders() bool
+	HasStorages() bool
 }
 
 func NewParser[T Configuration](envPrefix string, configFilename string, configSearchPaths ...string) Parser[T] {
@@ -48,7 +48,7 @@ func (p Parser[T]) ReadConfig(config T, file string) error {
 		return fmt.Errorf("could not unmarshal configuration: %s", err)
 	}
 
-	if !config.HasUploaders() {
+	if !config.HasStorages() {
 		return fmt.Errorf("no uploaders configured")
 	}
 

diff --git a/internal/agent/config/config_test.go b/internal/agent/config/config_test.go
@@ -25,7 +25,7 @@ type configDataStub struct {
 	}
 }
 
-func (stub configDataStub) HasUploaders() bool {
+func (stub configDataStub) HasStorages() bool {
 	return stub.hasUploaders
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -25,7 +25,7 @@ type configDataStub struct { @@
     	}
     }
-    func (stub configDataStub) HasUploaders() bool {
+    func (stub configDataStub) HasStorages() bool {
     	return stub.hasUploaders
     }
@@ Expand Down @@