Added abstract helper interface for all storage backends (#135)

* Added abstract helper interface and implemented it for all storage backends

* Moved storage client initializations also to helper classes

* Fixed ssh init issue

* Moved script parameter to helper struct to simplify script init.

* Created sub modules. Enhanced abstract implementation.

* Fixed config issue

* Fixed declaration issues. Added config to interface.

* Added StorageProviders to unify all backends.

* Cleanup, optimizations, comments.

* Applied discussed changes. See description.

Moved modules to internal packages.
Replaced StoragePool with slice.
Moved conditional for init of storage backends back to script.

* Fix docker build issue

* Fixed accidentally removed local copy condition.

* Delete .gitignore

* Renaming/changes according to review

Renamed Init functions and interface.
Replaced config object with specific config values.
Init func returns interface instead of struct.
Removed custom import names where possible.

* Fixed auto-complete error.

* Combined copy instructions into one layer.

* Added logging func for storages.

* Introduced logging func for errors too.

* Missed an error message

* Moved config back to main. Optimized prune stats handling.

* Move stats back to main package

* Code doc stuff

* Apply changes from #136

* Replace name field with function.

* Changed receiver names from stg to b.

* Renamed LogFuncDef to Log

* Removed redundant package name.

* Renamed storagePool to storages.

* Simplified creation of new storage backend.

* Added initialization for storage stats map.

* Invert .dockerignore patterns.

* Fix package typo

.dockerignore

@@ -1 +1,19 @@
-test
+# Ignore everything
+*
+
+# Exceptions:
+# Note: Wildcards for directories like * or ** don't work (yet) with exclamation marks!
+
+!cmd/backup/*.go
+!cmd/backup/*.tmpl
+
+!internal/storage/*.go
+!internal/storage/local/*.go
+!internal/storage/s3/*.go
+!internal/storage/ssh/*.go
+!internal/storage/webdav/*.go
+!internal/utilities/*.go
+
+!Dockerfile
+!entrypoint.sh
+!go.*

Dockerfile

@@ -4,9 +4,8 @@
 FROM golang:1.18-alpine as builder
 
 WORKDIR /app
-COPY go.mod go.sum ./
+COPY . .
 RUN go mod download
-COPY cmd/backup ./cmd/backup/
 WORKDIR /app/cmd/backup
 RUN go build -o backup .
 

cmd/backup/config.go

@@ -12,6 +12,15 @@ import (
 // Config holds all configuration values that are expected to be set
 // by users.
 type Config struct {
+	AwsS3BucketName            string        `split_words:"true"`
+	AwsS3Path                  string        `split_words:"true"`
+	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
+	AwsEndpointProto           string        `split_words:"true" default:"https"`
+	AwsEndpointInsecure        bool          `split_words:"true"`
+	AwsStorageClass            string        `split_words:"true"`
+	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
+	AwsSecretAccessKey         string        `split_words:"true"`
+	AwsIamRoleEndpoint         string        `split_words:"true"`
 	BackupSources              string        `split_words:"true" default:"/backup"`
 	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
 	BackupFilenameExpand       bool          `split_words:"true"`
@@ -23,15 +32,6 @@ type Config struct {
 	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
 	BackupFromSnapshot         bool          `split_words:"true"`
 	BackupExcludeRegexp        RegexpDecoder `split_words:"true"`
-	AwsS3BucketName            string        `split_words:"true"`
-	AwsS3Path                  string        `split_words:"true"`
-	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
-	AwsEndpointProto           string        `split_words:"true" default:"https"`
-	AwsEndpointInsecure        bool          `split_words:"true"`
-	AwsStorageClass            string        `split_words:"true"`
-	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
-	AwsSecretAccessKey         string        `split_words:"true"`
-	AwsIamRoleEndpoint         string        `split_words:"true"`
 	GpgPassphrase              string        `split_words:"true"`
 	NotificationURLs           []string      `envconfig:"NOTIFICATION_URLS"`
 	NotificationLevel          string        `split_words:"true" default:"error"`

cmd/backup/hooks.go

@@ -6,6 +6,8 @@ package main
 import (
 	"fmt"
 	"sort"
+
+	"github.com/offen/docker-volume-backup/internal/utilities"
 )
 
 // hook contains a queued action that can be trigger them when the script
@@ -50,7 +52,7 @@ func (s *script) runHooks(err error) error {
 		}
 	}
 	if len(actionErrors) != 0 {
-		return join(actionErrors...)
+		return utilities.Join(actionErrors...)
 	}
 	return nil
 }

cmd/backup/lock.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/gofrs/flock"
+	"github.com/offen/docker-volume-backup/internal/utilities"
 )
 
 // lock opens a lockfile at the given location, keeping it locked until the
@@ -31,7 +32,7 @@ func (s *script) lock(lockfile string) (func() error, error) {
 	for {
 		acquired, err := fileLock.TryLock()
 		if err != nil {
-			return noop, fmt.Errorf("lock: error trying lock: %w", err)
+			return utilities.Noop, fmt.Errorf("lock: error trying lock: %w", err)
 		}
 		if acquired {
 			if s.encounteredLock {
@@ -52,7 +53,7 @@ func (s *script) lock(lockfile string) (func() error, error) {
 		case <-retry.C:
 			continue
 		case <-deadline.C:
-			return noop, errors.New("lock: timed out waiting for lockfile to become available")
+			return utilities.Noop, errors.New("lock: timed out waiting for lockfile to become available")
 		}
 	}
 }

cmd/backup/notifications.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	sTypes "github.com/containrrr/shoutrrr/pkg/types"
+	"github.com/offen/docker-volume-backup/internal/utilities"
 )
 
 //go:embed notifications.tmpl
@@ -68,7 +69,7 @@ func (s *script) sendNotification(title, body string) error {
 		}
 	}
 	if len(errs) != 0 {
-		return fmt.Errorf("sendNotification: error sending message: %w", join(errs...))
+		return fmt.Errorf("sendNotification: error sending message: %w", utilities.Join(errs...))
 	}
 	return nil
 }