From d0a63dafb0d2ffc95f05a8edf965da7eb2311b7a Mon Sep 17 00:00:00 2001 From: Michal Middleton Date: Fri, 18 Aug 2023 08:12:29 -0500 Subject: [PATCH] Add support for zstd compression --- README.md | 17 ++++++++--- cmd/backup/archive.go | 44 +++++++++++++++++++---------- cmd/backup/config.go | 17 ++++++++++- cmd/backup/script.go | 19 ++++++++++++- go.mod | 2 +- test/cli-zstd/run.sh | 66 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 143 insertions(+), 22 deletions(-) create mode 100755 test/cli-zstd/run.sh diff --git a/README.md b/README.md index df226acb..bab2a443 100644 --- a/README.md +++ b/README.md @@ -148,13 +148,22 @@ You can populate below template according to your requirements and use it as you # BACKUP_CRON_EXPRESSION="0 2 * * *" -# The name of the backup file including the `.tar.gz` extension. +# The compression algorithm used in conjunction with tar. +# Valid options are: "gz" and "zst". +# Note that the selection affects the file extension. + +# BACKUP_COMPRESSION="gz" + +# The name of the backup file including the extension. # Format verbs will be replaced as in `strftime`. Omitting them # will result in the same filename for every backup run, which means previous -# versions will be overwritten on subsequent runs. The default results -# in filenames like `backup-2021-08-29T04-00-00.tar.gz`. +# versions will be overwritten on subsequent runs. +# Extension can be defined literally or via "{{ .Extension }}" template, +# in which case it will become either "tar.gz" or "tar.zst" (depending +# on your BACKUP_COMPRESSION setting). +# The default results in filenames like: `backup-2021-08-29T04-00-00.tar.gz`. -# BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.tar.gz" +# BACKUP_FILENAME="backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" # Setting BACKUP_FILENAME_EXPAND to true allows for environment variable # placeholders in BACKUP_FILENAME, BACKUP_LATEST_SYMLINK and in diff --git a/cmd/backup/archive.go b/cmd/backup/archive.go index 513df978..8dd4c4ad 100644 --- a/cmd/backup/archive.go +++ b/cmd/backup/archive.go @@ -15,9 +15,11 @@ import ( "path" "path/filepath" "strings" + + "github.com/klauspost/compress/zstd" ) -func createArchive(files []string, inputFilePath, outputFilePath string) error { +func createArchive(files []string, inputFilePath, outputFilePath string, compression string) error { inputFilePath = stripTrailingSlashes(inputFilePath) inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath) if err != nil { @@ -27,7 +29,7 @@ func createArchive(files []string, inputFilePath, outputFilePath string) error { return fmt.Errorf("createArchive: error creating output file path: %w", err) } - if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil { + if err := compress(files, outputFilePath, filepath.Dir(inputFilePath), compression); err != nil { return fmt.Errorf("createArchive: error creating archive: %w", err) } @@ -51,18 +53,30 @@ func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) return inputFilePath, outputFilePath, err } -func compress(paths []string, outFilePath, subPath string) error { +func compress(paths []string, outFilePath, subPath string, algo string) error { file, err := os.Create(outFilePath) + var compressWriter io.WriteCloser if err != nil { return fmt.Errorf("compress: error creating out file: %w", err) } prefix := path.Dir(outFilePath) - gzipWriter := gzip.NewWriter(file) - tarWriter := tar.NewWriter(gzipWriter) + switch algo { + case "gz": + compressWriter = gzip.NewWriter(file) + case "zst": + compressWriter, err = zstd.NewWriter(file) + if err != nil { + return fmt.Errorf("compress: zstd error: %w", err) + } + default: + return fmt.Errorf("compress: unsupported compression algorithm: %s", algo) + } + + tarWriter := tar.NewWriter(compressWriter) for _, p := range paths { - if err := writeTarGz(p, tarWriter, prefix); err != nil { + if err := writeTarball(p, tarWriter, prefix); err != nil { return fmt.Errorf("compress: error writing %s to archive: %w", p, err) } } @@ -72,9 +86,9 @@ func compress(paths []string, outFilePath, subPath string) error { return fmt.Errorf("compress: error closing tar writer: %w", err) } - err = gzipWriter.Close() + err = compressWriter.Close() if err != nil { - return fmt.Errorf("compress: error closing gzip writer: %w", err) + return fmt.Errorf("compress: error closing compression writer: %w", err) } err = file.Close() @@ -85,10 +99,10 @@ func compress(paths []string, outFilePath, subPath string) error { return nil } -func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error { +func writeTarball(path string, tarWriter *tar.Writer, prefix string) error { fileInfo, err := os.Lstat(path) if err != nil { - return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err) + return fmt.Errorf("writeTarball: error getting file infor for %s: %w", path, err) } if fileInfo.Mode()&os.ModeSocket == os.ModeSocket { @@ -99,19 +113,19 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error { if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { var err error if link, err = os.Readlink(path); err != nil { - return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err) + return fmt.Errorf("writeTarball: error resolving symlink %s: %w", path, err) } } header, err := tar.FileInfoHeader(fileInfo, link) if err != nil { - return fmt.Errorf("writeTarGz: error getting file info header: %w", err) + return fmt.Errorf("writeTarball: error getting file info header: %w", err) } header.Name = strings.TrimPrefix(path, prefix) err = tarWriter.WriteHeader(header) if err != nil { - return fmt.Errorf("writeTarGz: error writing file info header: %w", err) + return fmt.Errorf("writeTarball: error writing file info header: %w", err) } if !fileInfo.Mode().IsRegular() { @@ -120,13 +134,13 @@ func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error { file, err := os.Open(path) if err != nil { - return fmt.Errorf("writeTarGz: error opening %s: %w", path, err) + return fmt.Errorf("writeTarball: error opening %s: %w", path, err) } defer file.Close() _, err = io.Copy(tarWriter, file) if err != nil { - return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err) + return fmt.Errorf("writeTarball: error copying %s to tar writer: %w", path, err) } return nil diff --git a/cmd/backup/config.go b/cmd/backup/config.go index d3825e80..a565c298 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -29,8 +29,9 @@ type Config struct { AwsSecretAccessKeyFile string `split_words:"true"` AwsIamRoleEndpoint string `split_words:"true"` AwsPartSize int64 `split_words:"true"` + BackupCompression string `split_words:"true" default:"gz"` BackupSources string `split_words:"true" default:"/backup"` - BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` + BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"` BackupFilenameExpand bool `split_words:"true"` BackupLatestSymlink string `split_words:"true"` BackupArchive string `split_words:"true" default:"/archive"` @@ -82,6 +83,16 @@ func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) return string(data), nil } +func (c *Config) SetCompressionAlgo(algo string) error { + switch algo { + case "gz", "zst": + c.BackupCompression = algo + return nil + default: + return fmt.Errorf("config: unknown BACKUP_COMPRESSION %s", algo) + } +} + type CertDecoder struct { Cert *x509.Certificate } @@ -118,3 +129,7 @@ func (r *RegexpDecoder) Decode(v string) error { *r = RegexpDecoder{Re: re} return nil } + +type CompAlgo struct { + Str *string +} diff --git a/cmd/backup/script.go b/cmd/backup/script.go index d8823808..6beae4ce 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "errors" "fmt" @@ -89,6 +90,21 @@ func newScript() (*script, error) { } s.file = path.Join("/tmp", s.c.BackupFilename) + + extMap := map[string]string{ + "Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression), + } + + tmplFileName, tErr := template.New("extension").Parse(s.file) + if tErr != nil { + return nil, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr) + } + var bf bytes.Buffer + if tErr := tmplFileName.Execute(&bf, extMap); tErr != nil { + return nil, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr) + } + s.file = bf.String() + if s.c.BackupFilenameExpand { s.file = os.ExpandEnv(s.file) s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink) @@ -424,6 +440,7 @@ func (s *script) createArchive() error { } tarFile := s.file + compressionAlgo := s.c.BackupCompression s.registerHook(hookLevelPlumbing, func(error) error { if err := remove(tarFile); err != nil { return fmt.Errorf("createArchive: error removing tar file: %w", err) @@ -454,7 +471,7 @@ func (s *script) createArchive() error { return fmt.Errorf("createArchive: error walking filesystem tree: %w", err) } - if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil { + if err := createArchive(filesEligibleForBackup, backupSources, tarFile, compressionAlgo); err != nil { return fmt.Errorf("createArchive: error compressing backup folder: %w", err) } diff --git a/go.mod b/go.mod index 2a8ade29..3b752929 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/docker/docker v24.0.5+incompatible github.com/gofrs/flock v0.8.1 github.com/kelseyhightower/envconfig v1.4.0 + github.com/klauspost/compress v1.16.7 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d github.com/minio/minio-go/v7 v7.0.61 github.com/otiai10/copy v1.11.0 @@ -33,7 +34,6 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/test/cli-zstd/run.sh b/test/cli-zstd/run.sh new file mode 100755 index 00000000..c7a0c9eb --- /dev/null +++ b/test/cli-zstd/run.sh @@ -0,0 +1,66 @@ +#!/bin/sh + +set -e + +cd $(dirname $0) +. ../util.sh +current_test=$(basename $(pwd)) + +docker network create test_network +docker volume create backup_data +docker volume create app_data +# This volume is created to test whether empty directories are handled +# correctly. It is not supposed to hold any data. +docker volume create empty_data + +docker run -d \ + --name minio \ + --network test_network \ + --env MINIO_ROOT_USER=test \ + --env MINIO_ROOT_PASSWORD=test \ + --env MINIO_ACCESS_KEY=test \ + --env MINIO_SECRET_KEY=GMusLtUmILge2by+z890kQ \ + -v backup_data:/data \ + minio/minio:RELEASE.2020-08-04T23-10-51Z server /data + +docker exec minio mkdir -p /data/backup + +docker run -d \ + --name offen \ + --network test_network \ + -v app_data:/var/opt/offen/ \ + offen/offen:latest + +sleep 10 + +docker run --rm \ + --network test_network \ + -v app_data:/backup/app_data \ + -v empty_data:/backup/empty_data \ + -v /var/run/docker.sock:/var/run/docker.sock \ + --env AWS_ACCESS_KEY_ID=test \ + --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \ + --env AWS_ENDPOINT=minio:9000 \ + --env AWS_ENDPOINT_PROTO=http \ + --env AWS_S3_BUCKET_NAME=backup \ + --env BACKUP_COMPRESSION=zst \ + --env BACKUP_FILENAME='test.{{ .Extension }}' \ + --env "BACKUP_FROM_SNAPSHOT=true" \ + --entrypoint backup \ + offen/docker-volume-backup:${TEST_VERSION:-canary} + +# Have to install tar and zstd on Alpine because the plain image comes with very +# basic tar from busybox and it does not seem to support zstd +docker run --rm \ + -v backup_data:/data alpine \ + ash -c 'apk add --no-cache zstd tar && tar -xvf /data/backup/test.tar.zst --zstd && test -f /backup/app_data/offen.db && test -d /backup/empty_data' + +pass "Found relevant files in untared remote backup." + +# This test does not stop containers during backup. This is happening on +# purpose in order to cover this setup as well. +expect_running_containers "2" + +docker rm $(docker stop minio offen) +docker volume rm backup_data app_data +docker network rm test_network