diff --git a/README.md b/README.md index 09e9ab7..a2bb2af 100644 --- a/README.md +++ b/README.md @@ -383,10 +383,15 @@ uploaders: accountKey: container: cloudDomain: - google: + gcp: bucket: local: path: + swift: + container: + username: + apiKey: + authUrl: ``` Note that if you specify more than one storage option, *all* options will be written to. For example, specifying `local` and `aws` will write to both locations. Each options can be specified exactly once - thus is is currently not possible to e.g. upload to multiple aws regions by specifying multiple `aws`-entries. @@ -420,13 +425,24 @@ uploaders: - `cloudDomain` *(default: blob.core.windows.net) - domain of the cloud-service to use -#### Google Storage +#### Google Cloud Storage `bucket` **(required)** - the Google Storage Bucket to write to. Auth is expected to be default machine credentials. #### Local Storage `path` **(required)** - fully qualified path, not including file name, for where the snapshot should be written. i.e. `/raft/snapshots` +#### Openstack Swift Storage +- `container` **(required)** - the name of the container to write to +- `username` **(required)** - the username used for authentication +- `apiKey` **(required)** - the api-key used for authentication +- `authUrl` **(required)** - the auth-url to authenicate against +- `region` - optional region to use eg "LON", "ORD" +- `domain` - optional user's domain name +- `tenantId` - optional id of the tenant +- `timeout` *(default: 60s)** - timeout for snapshot-uploads + + ## License - Source code is licensed under MIT @@ -434,4 +450,5 @@ uploaders: ## Contributors - Vault Raft Snapshot Agent was originally developed by [@Lucretius](https://github.com/Lucretius/vault_raft_snapshot_agent/) - This build contains improvements done by [@Boostport](https://github.com/Boostport/vault_raft_snapshot_agent/) -- support for additional authentication methods based on code from [@alexeiser](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/25) \ No newline at end of file +- support for additional authentication methods based on code from [@alexeiser](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/25) +- support for Openstack Swift Storage based on code from [@Pyjou](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/19) \ No newline at end of file diff --git a/cmd/vault-raft-snapshot-agent/main.go b/cmd/vault-raft-snapshot-agent/main.go index 629e073..12aba23 100644 --- a/cmd/vault-raft-snapshot-agent/main.go +++ b/cmd/vault-raft-snapshot-agent/main.go @@ -105,15 +105,15 @@ Options: } func startSnapshotter(configFile cli.Path) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + snapshotterOptions.ConfigFilePath = configFile - snapshotter, err := internal.CreateSnapshotter(snapshotterOptions) + snapshotter, err := internal.CreateSnapshotter(ctx, snapshotterOptions) if err != nil { log.Fatalf("Cannot create snapshotter: %s\n", err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { diff --git a/go.mod b/go.mod index 6bd7e5d..b6db967 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,9 @@ require ( google.golang.org/api v0.140.0 ) +// Swift-Uploader +require github.com/ncw/swift/v2 v2.0.2 + // Vault require ( github.com/hashicorp/vault/api v1.10.0 diff --git a/go.sum b/go.sum index f865e6c..7fc69b9 100644 --- a/go.sum +++ b/go.sum @@ -392,6 +392,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= +github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= diff --git a/internal/app/vault_raft_snapshot_agent/snapshotter.go b/internal/app/vault_raft_snapshot_agent/snapshotter.go index 173571a..9a9cd48 100644 --- a/internal/app/vault_raft_snapshot_agent/snapshotter.go +++ b/internal/app/vault_raft_snapshot_agent/snapshotter.go @@ -54,15 +54,15 @@ type snapshotterVaultAPI interface { TakeSnapshot(ctx context.Context, writer io.Writer) error } -func CreateSnapshotter(options SnapshotterOptions) (*Snapshotter, error) { - c := SnapshotterConfig{} +func CreateSnapshotter(ctx context.Context, options SnapshotterOptions) (*Snapshotter, error) { + data := SnapshotterConfig{} parser := config.NewParser[*SnapshotterConfig](options.ConfigFileName, options.EnvPrefix, options.ConfigFileSearchPaths...) - if err := parser.ReadConfig(&c, options.ConfigFilePath); err != nil { + if err := parser.ReadConfig(&data, options.ConfigFilePath); err != nil { return nil, err } - snapshotter, err := createSnapshotter(c) + snapshotter, err := createSnapshotter(ctx, data) if err != nil { return nil, err } @@ -70,7 +70,7 @@ func CreateSnapshotter(options SnapshotterOptions) (*Snapshotter, error) { parser.OnConfigChange( &SnapshotterConfig{}, func(config *SnapshotterConfig) error { - if err := snapshotter.reconfigure(*config); err != nil { + if err := snapshotter.reconfigure(ctx, *config); err != nil { log.Printf("could not reconfigure snapshotter: %s\n", err) return err } @@ -81,20 +81,20 @@ func CreateSnapshotter(options SnapshotterOptions) (*Snapshotter, error) { return snapshotter, nil } -func createSnapshotter(config SnapshotterConfig) (*Snapshotter, error) { +func createSnapshotter(ctx context.Context, config SnapshotterConfig) (*Snapshotter, error) { snapshotter := &Snapshotter{} - err := snapshotter.reconfigure(config) + err := snapshotter.reconfigure(ctx, config) return snapshotter, err } -func (s *Snapshotter) reconfigure(config SnapshotterConfig) error { +func (s *Snapshotter) reconfigure(ctx context.Context, config SnapshotterConfig) error { client, err := vault.CreateVaultClient(config.Vault) if err != nil { return err } - uploaders, err := upload.CreateUploaders(config.Uploaders) + uploaders, err := upload.CreateUploaders(ctx, config.Uploaders) if err != nil { return err } diff --git a/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go b/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go index 59601a9..19ebc9a 100644 --- a/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go +++ b/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go @@ -125,6 +125,16 @@ func TestReadCompleteConfig(t *testing.T) { Local: upload.LocalUploaderConfig{ Path: ".", }, + Swift: upload.SwiftUploaderConfig{ + Container: "test-container", + UserName: "test-username", + ApiKey: "test-api-key", + AuthUrl: "http://auth.com", + Domain: "http://user.com", + Region: "test-region", + TenantId: "test-tenant", + Timeout: 180 * time.Second, + }, }, } @@ -198,6 +208,10 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { Local: upload.LocalUploaderConfig{ Path: ".", }, + Swift: upload.SwiftUploaderConfig{ + Timeout: time.Minute, + Empty: true, + }, }, } diff --git a/internal/app/vault_raft_snapshot_agent/upload/aws.go b/internal/app/vault_raft_snapshot_agent/upload/aws.go index fa481d7..38140ac 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/aws.go +++ b/internal/app/vault_raft_snapshot_agent/upload/aws.go @@ -39,8 +39,8 @@ type awsUploaderImpl struct { sse bool } -func createAWSUploader(config AWSUploaderConfig) (*uploader[s3Types.Object], error) { - clientConfig, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithRegion(config.Region)) +func createAWSUploader(ctx context.Context, config AWSUploaderConfig) (*uploader[s3Types.Object], error) { + clientConfig, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithRegion(config.Region)) if err != nil { return nil, fmt.Errorf("failed to load default aws config: %w", err) diff --git a/internal/app/vault_raft_snapshot_agent/upload/azure.go b/internal/app/vault_raft_snapshot_agent/upload/azure.go index e6083d5..720cb07 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/azure.go +++ b/internal/app/vault_raft_snapshot_agent/upload/azure.go @@ -22,7 +22,7 @@ type azureUploaderImpl struct { container string } -func createAzureUploader(config AzureUploaderConfig) (*uploader[*container.BlobItem], error) { +func createAzureUploader(ctx context.Context, config AzureUploaderConfig) (*uploader[*container.BlobItem], error) { credential, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey) if err != nil { return nil, fmt.Errorf("invalid credentials for azure: %w", err) diff --git a/internal/app/vault_raft_snapshot_agent/upload/gcp.go b/internal/app/vault_raft_snapshot_agent/upload/gcp.go index 8a5cea8..18a59c0 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/gcp.go +++ b/internal/app/vault_raft_snapshot_agent/upload/gcp.go @@ -19,8 +19,7 @@ type gcpUploaderImpl struct { bucket *storage.BucketHandle } -func createGCPUploader(config GCPUploaderConfig) (*uploader[storage.ObjectAttrs], error) { - ctx := context.Background() +func createGCPUploader(ctx context.Context, config GCPUploaderConfig) (*uploader[storage.ObjectAttrs], error) { client, err := storage.NewClient(ctx) if err != nil { return nil, err @@ -42,7 +41,7 @@ func (u gcpUploaderImpl) Destination() string { // implements interface uploaderImpl func (u gcpUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { obj := u.bucket.Object(name) - w := obj.NewWriter(context.Background()) + w := obj.NewWriter(ctx) if _, err := io.Copy(w, data); err != nil { return err diff --git a/internal/app/vault_raft_snapshot_agent/upload/local.go b/internal/app/vault_raft_snapshot_agent/upload/local.go index 7de73b6..0029216 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/local.go +++ b/internal/app/vault_raft_snapshot_agent/upload/local.go @@ -17,7 +17,7 @@ type localUploaderImpl struct { path string } -func createLocalUploader(config LocalUploaderConfig) (uploader[os.FileInfo], error) { +func createLocalUploader(ctx context.Context, config LocalUploaderConfig) (uploader[os.FileInfo], error) { return uploader[os.FileInfo]{ localUploaderImpl{ path: config.Path, diff --git a/internal/app/vault_raft_snapshot_agent/upload/swift.go b/internal/app/vault_raft_snapshot_agent/upload/swift.go new file mode 100644 index 0000000..a2c5f06 --- /dev/null +++ b/internal/app/vault_raft_snapshot_agent/upload/swift.go @@ -0,0 +1,106 @@ +package upload + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/ncw/swift/v2" +) + +type SwiftUploaderConfig struct { + Container string `validate:"required_if=Empty false"` + UserName string `validate:"required_if=Empty false"` + ApiKey string `validate:"required_if=Empty false"` + AuthUrl string `validate:"required_if=Empty false,omitempty,http_url"` + Domain string `validate:"omitempty,http_url"` + Region string + TenantId string + Timeout time.Duration `default:"60s"` + Empty bool +} + +type swiftUploaderImpl struct { + connection *swift.Connection + container string +} + +func createSwiftUploader(ctx context.Context, config SwiftUploaderConfig) (*uploader[swift.Object], error) { + conn := swift.Connection{ + UserName: config.UserName, + ApiKey: config.ApiKey, + AuthUrl: config.AuthUrl, + Region: config.Region, + TenantId: config.TenantId, + Domain: config.Domain, + Timeout: config.Timeout, + } + + if err := conn.Authenticate(ctx); err != nil { + return nil, fmt.Errorf("invalid credentials: %s", err) + } + + if _, _, err := conn.Container(ctx, config.Container); err != nil { + return nil, fmt.Errorf("invalid container %s: %s", config.Container, err) + } + + return &uploader[swift.Object]{ + swiftUploaderImpl{ + connection: &conn, + container: config.Container, + }, + }, nil +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) Destination() string { + return fmt.Sprintf("swift container %s", u.container) +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { + _, header, err := u.connection.Container(ctx, u.container) + if err != nil { + return err + } + + object, err := u.connection.ObjectCreate(ctx, u.container, name, false, "", "", header) + if err != nil { + return err + } + + if _, err := io.Copy(object, data); err != nil { + return err + } + + if err := object.Close(); err != nil { + return err + } + + return nil +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) deleteSnapshot(ctx context.Context, snapshot swift.Object) error { + if err := u.connection.ObjectDelete(ctx, u.container, snapshot.Name); err != nil { + return err + } + + return nil +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]swift.Object, error) { + return u.connection.ObjectsAll(ctx, u.container, &swift.ObjectsOpts{Prefix: prefix}) +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) compareSnapshots(a, b swift.Object) int { + return a.LastModified.Compare(a.LastModified) +} diff --git a/internal/app/vault_raft_snapshot_agent/upload/uploaders.go b/internal/app/vault_raft_snapshot_agent/upload/uploaders.go index ba8a593..4b09dd4 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/uploaders.go +++ b/internal/app/vault_raft_snapshot_agent/upload/uploaders.go @@ -9,10 +9,11 @@ import ( ) type UploadersConfig struct { - AWS AWSUploaderConfig `default:"{\"Empty\": true}" mapstructure:"aws"` - Azure AzureUploaderConfig `default:"{\"Empty\": true}" mapstructure:"azure"` - GCP GCPUploaderConfig `default:"{\"Empty\": true}" mapstructure:"google"` - Local LocalUploaderConfig `default:"{\"Empty\": true}" mapstructure:"local"` + AWS AWSUploaderConfig `default:"{\"Empty\": true}"` + Azure AzureUploaderConfig `default:"{\"Empty\": true}"` + GCP GCPUploaderConfig `default:"{\"Empty\": true}"` + Local LocalUploaderConfig `default:"{\"Empty\": true}"` + Swift SwiftUploaderConfig `default:"{\"Empty\": true}"` } type Uploader interface { @@ -20,11 +21,11 @@ type Uploader interface { Upload(ctx context.Context, snapshot io.Reader, prefix string, timestamp string, suffix string, retain int) error } -func CreateUploaders(config UploadersConfig) ([]Uploader, error) { +func CreateUploaders(ctx context.Context, config UploadersConfig) ([]Uploader, error) { var uploaders []Uploader if !config.AWS.Empty { - aws, err := createAWSUploader(config.AWS) + aws, err := createAWSUploader(ctx, config.AWS) if err != nil { return nil, err } @@ -32,7 +33,7 @@ func CreateUploaders(config UploadersConfig) ([]Uploader, error) { } if !config.Azure.Empty { - azure, err := createAzureUploader(config.Azure) + azure, err := createAzureUploader(ctx, config.Azure) if err != nil { return nil, err } @@ -40,7 +41,7 @@ func CreateUploaders(config UploadersConfig) ([]Uploader, error) { } if !config.GCP.Empty { - gcp, err := createGCPUploader(config.GCP) + gcp, err := createGCPUploader(ctx, config.GCP) if err != nil { return nil, err } @@ -48,7 +49,15 @@ func CreateUploaders(config UploadersConfig) ([]Uploader, error) { } if !config.Local.Empty { - local, err := createLocalUploader(config.Local) + local, err := createLocalUploader(ctx, config.Local) + if err != nil { + return nil, err + } + uploaders = append(uploaders, local) + } + + if !config.Swift.Empty { + local, err := createSwiftUploader(ctx, config.Swift) if err != nil { return nil, err } diff --git a/testdata/complete.yaml b/testdata/complete.yaml index 72ab197..0939e91 100644 --- a/testdata/complete.yaml +++ b/testdata/complete.yaml @@ -57,7 +57,17 @@ uploaders: accountKey: test-key container: test-container cloudDomain: blob.core.chinacloudapi.cn - google: + gcp: bucket: test-bucket local: path: . + swift: + container: test-container + username: test-username + apiKey: test-api-key + authUrl: http://auth.com + domain: http://user.com + region: test-region + tenantId: test-tenant + timeout: 180s +