From a931eef0612dcfd124ce90704641f73d110a289c Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 23 Dec 2022 09:21:26 +0100 Subject: [PATCH 01/11] Scaffold Azure storage backend that does nothing yet --- cmd/backup/config.go | 97 +++++++++++++++++---------------- cmd/backup/script.go | 15 +++++ go.mod | 3 + go.sum | 7 +++ internal/storage/azure/azure.go | 42 ++++++++++++++ 5 files changed, 117 insertions(+), 47 deletions(-) create mode 100644 internal/storage/azure/azure.go diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 8743cf48..d2fea130 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -16,53 +16,56 @@ import ( // Config holds all configuration values that are expected to be set // by users. type Config struct { - AwsS3BucketName string `split_words:"true"` - AwsS3Path string `split_words:"true"` - AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` - AwsEndpointProto string `split_words:"true" default:"https"` - AwsEndpointInsecure bool `split_words:"true"` - AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` - AwsStorageClass string `split_words:"true"` - AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` - AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"` - AwsSecretAccessKey string `split_words:"true"` - AwsSecretAccessKeyFile string `split_words:"true"` - AwsIamRoleEndpoint string `split_words:"true"` - BackupSources string `split_words:"true" default:"/backup"` - BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` - BackupFilenameExpand bool `split_words:"true"` - BackupLatestSymlink string `split_words:"true"` - BackupArchive string `split_words:"true" default:"/archive"` - BackupRetentionDays int32 `split_words:"true" default:"-1"` - BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` - BackupPruningPrefix string `split_words:"true"` - BackupStopContainerLabel string `split_words:"true" default:"true"` - BackupFromSnapshot bool `split_words:"true"` - BackupExcludeRegexp RegexpDecoder `split_words:"true"` - GpgPassphrase string `split_words:"true"` - NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` - NotificationLevel string `split_words:"true" default:"error"` - EmailNotificationRecipient string `split_words:"true"` - EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` - EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` - EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` - EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` - EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` - WebdavUrl string `split_words:"true"` - WebdavUrlInsecure bool `split_words:"true"` - WebdavPath string `split_words:"true" default:"/"` - WebdavUsername string `split_words:"true"` - WebdavPassword string `split_words:"true"` - SSHHostName string `split_words:"true"` - SSHPort string `split_words:"true" default:"22"` - SSHUser string `split_words:"true"` - SSHPassword string `split_words:"true"` - SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` - SSHIdentityPassphrase string `split_words:"true"` - SSHRemotePath string `split_words:"true"` - ExecLabel string `split_words:"true"` - ExecForwardOutput bool `split_words:"true"` - LockTimeout time.Duration `split_words:"true" default:"60m"` + AwsS3BucketName string `split_words:"true"` + AwsS3Path string `split_words:"true"` + AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` + AwsEndpointProto string `split_words:"true" default:"https"` + AwsEndpointInsecure bool `split_words:"true"` + AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` + AwsStorageClass string `split_words:"true"` + AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` + AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"` + AwsSecretAccessKey string `split_words:"true"` + AwsSecretAccessKeyFile string `split_words:"true"` + AwsIamRoleEndpoint string `split_words:"true"` + BackupSources string `split_words:"true" default:"/backup"` + BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` + BackupFilenameExpand bool `split_words:"true"` + BackupLatestSymlink string `split_words:"true"` + BackupArchive string `split_words:"true" default:"/archive"` + BackupRetentionDays int32 `split_words:"true" default:"-1"` + BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` + BackupPruningPrefix string `split_words:"true"` + BackupStopContainerLabel string `split_words:"true" default:"true"` + BackupFromSnapshot bool `split_words:"true"` + BackupExcludeRegexp RegexpDecoder `split_words:"true"` + GpgPassphrase string `split_words:"true"` + NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` + NotificationLevel string `split_words:"true" default:"error"` + EmailNotificationRecipient string `split_words:"true"` + EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` + EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` + EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` + EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` + EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` + WebdavUrl string `split_words:"true"` + WebdavUrlInsecure bool `split_words:"true"` + WebdavPath string `split_words:"true" default:"/"` + WebdavUsername string `split_words:"true"` + WebdavPassword string `split_words:"true"` + SSHHostName string `split_words:"true"` + SSHPort string `split_words:"true" default:"22"` + SSHUser string `split_words:"true"` + SSHPassword string `split_words:"true"` + SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` + SSHIdentityPassphrase string `split_words:"true"` + SSHRemotePath string `split_words:"true"` + ExecLabel string `split_words:"true"` + ExecForwardOutput bool `split_words:"true"` + LockTimeout time.Duration `split_words:"true" default:"60m"` + AzureStorageAccountName string `split_words:"true"` + AzureStoragePrimaryAccountKey string `split_words:"true"` + AzureStorageContainerName string `split_words:"true"` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/cmd/backup/script.go b/cmd/backup/script.go index ab7447b3..6ea5dd46 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -15,6 +15,7 @@ import ( "time" "github.com/offen/docker-volume-backup/internal/storage" + "github.com/offen/docker-volume-backup/internal/storage/azure" "github.com/offen/docker-volume-backup/internal/storage/local" "github.com/offen/docker-volume-backup/internal/storage/s3" "github.com/offen/docker-volume-backup/internal/storage/ssh" @@ -76,6 +77,7 @@ func newScript() (*script, error) { "WebDAV": {}, "SSH": {}, "Local": {}, + "Azure": {}, }, }, } @@ -189,6 +191,19 @@ func newScript() (*script, error) { s.storages = append(s.storages, localBackend) } + if s.c.AzureStorageAccountName != "" { + azureConfig := azure.Config{ + ContainerName: s.c.AzureStorageContainerName, + AccountName: s.c.AzureStorageAccountName, + PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey, + } + azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) + if err != nil { + return nil, err + } + s.storages = append(s.storages, azureBackend) + } + if s.c.EmailNotificationRecipient != "" { emailURL := fmt.Sprintf( "smtp://%s:%s@%s:%d/?from=%s&to=%s", diff --git a/go.mod b/go.mod index 96e63699..3bd4ca12 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,9 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/containerd/containerd v1.6.6 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect diff --git a/go.sum b/go.sum index 50b063f7..5c9318f6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -298,6 +304,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go new file mode 100644 index 00000000..726979fa --- /dev/null +++ b/internal/storage/azure/azure.go @@ -0,0 +1,42 @@ +// Copyright 2022 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package azure + +import ( + "time" + + "github.com/offen/docker-volume-backup/internal/storage" +) + +type azureBlobStorage struct { + *storage.StorageBackend +} + +// Config contains values that define the configuration of an Azure Blob Storage. +type Config struct { + AccountName string + ContainerName string + PrimaryAccountKey string +} + +// NewStorageBackend creates and initializes a new S3/Minio storage backend. +func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { + storage := azureBlobStorage{} + return &storage, nil +} + +// Name returns the name of the storage backend +func (v *azureBlobStorage) Name() string { + return "Azure" +} + +// Copy copies the given file to the storage backend. +func (b *azureBlobStorage) Copy(file string) error { + return nil +} + +// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend. +func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { + return nil, nil +} From d09aacf3f0b0e747f47be457bf45299d6310734e Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 23 Dec 2022 10:14:37 +0100 Subject: [PATCH 02/11] Implement copy for Azure Blob Storage --- cmd/backup/config.go | 1 + internal/storage/azure/azure.go | 37 ++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/cmd/backup/config.go b/cmd/backup/config.go index d2fea130..f293bdca 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -66,6 +66,7 @@ type Config struct { AzureStorageAccountName string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"` AzureStorageContainerName string `split_words:"true"` + AzureStorageEndpoint string `split_words:"true" default:""https://%s.blob.core.windows.net/""` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 726979fa..10340769 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -4,13 +4,19 @@ package azure import ( + "context" + "fmt" + "os" "time" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/offen/docker-volume-backup/internal/storage" ) type azureBlobStorage struct { *storage.StorageBackend + client *azblob.Client + containerName string } // Config contains values that define the configuration of an Azure Blob Storage. @@ -18,21 +24,46 @@ type Config struct { AccountName string ContainerName string PrimaryAccountKey string + Endpoint string } -// NewStorageBackend creates and initializes a new S3/Minio storage backend. +// NewStorageBackend creates and initializes a new Azure Blob Storage backend. func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { - storage := azureBlobStorage{} + cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating shared Azure credential: %w", err) + } + client, err := azblob.NewClientWithSharedKeyCredential(fmt.Sprintf(opts.Endpoint, opts.AccountName), cred, nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) + } + storage := azureBlobStorage{ + client: client, + containerName: opts.ContainerName, + } return &storage, nil } // Name returns the name of the storage backend -func (v *azureBlobStorage) Name() string { +func (b *azureBlobStorage) Name() string { return "Azure" } // Copy copies the given file to the storage backend. func (b *azureBlobStorage) Copy(file string) error { + fileReader, err := os.Open(file) + if err != nil { + return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err) + } + _, err = b.client.UploadStream(context.TODO(), + b.containerName, + file, + fileReader, + nil, + ) + if err != nil { + return fmt.Errorf("(*azureBlobStorage).Copy: error uploading file %s: %w", file, err) + } return nil } From 6aeec7b72b9e2c875b1be76a669945fa3fd005ad Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 23 Dec 2022 10:31:25 +0100 Subject: [PATCH 03/11] Set up automated testing for Azure Storage --- cmd/backup/config.go | 2 +- cmd/backup/script.go | 1 + internal/storage/azure/azure.go | 6 ++-- test/azure/docker-compose.yml | 57 +++++++++++++++++++++++++++++++++ test/azure/run.sh | 40 +++++++++++++++++++++++ 5 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 test/azure/docker-compose.yml create mode 100644 test/azure/run.sh diff --git a/cmd/backup/config.go b/cmd/backup/config.go index f293bdca..09ef3d21 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -66,7 +66,7 @@ type Config struct { AzureStorageAccountName string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"` AzureStorageContainerName string `split_words:"true"` - AzureStorageEndpoint string `split_words:"true" default:""https://%s.blob.core.windows.net/""` + AzureStorageEndpoint string `split_words:"true" default:"https://%s.blob.core.windows.net/"` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 6ea5dd46..379df288 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -196,6 +196,7 @@ func newScript() (*script, error) { ContainerName: s.c.AzureStorageContainerName, AccountName: s.c.AzureStorageAccountName, PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey, + Endpoint: s.c.AzureStorageEndpoint, } azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) if err != nil { diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 10340769..82165ceb 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "path" "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" @@ -55,9 +56,10 @@ func (b *azureBlobStorage) Copy(file string) error { if err != nil { return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err) } + _, err = b.client.UploadStream(context.TODO(), b.containerName, - file, + path.Base(file), fileReader, nil, ) @@ -69,5 +71,5 @@ func (b *azureBlobStorage) Copy(file string) error { // Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend. func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { - return nil, nil + return &storage.PruneStats{}, nil } diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml new file mode 100644 index 00000000..5e3ff01b --- /dev/null +++ b/test/azure/docker-compose.yml @@ -0,0 +1,57 @@ +version: '3' + +services: + storage: + image: mcr.microsoft.com/azure-storage/azurite + volumes: + - ./foo:/data + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data + healthcheck: + test: nc 127.0.0.1 10000 -z + interval: 1s + retries: 30 + + az_cli: + image: mcr.microsoft.com/azure-cli + volumes: + - ./local:/dump + command: + - /bin/sh + - -c + - | + az storage container create --name test-container + depends_on: + storage: + condition: service_healthy + environment: + AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1; + + backup: + image: offen/docker-volume-backup:${TEST_VERSION:-canary} + hostname: hostnametoken + restart: always + environment: + AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1 + AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + AZURE_STORAGE_CONTAINER_NAME: test-container + AZURE_STORAGE_ENDPOINT: http://storage:10000/%s/ + BACKUP_FILENAME: test.tar.gz + BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_PRUNING_LEEWAY: 5s + BACKUP_PRUNING_PREFIX: test + volumes: + - app_data:/backup/app_data:ro + - /var/run/docker.sock:/var/run/docker.sock + + offen: + image: offen/offen:latest + labels: + - docker-volume-backup.stop-during-backup=true + volumes: + - app_data:/var/opt/offen + +volumes: + azurite_backup_data: + name: azurite_backup_data + app_data: diff --git a/test/azure/run.sh b/test/azure/run.sh new file mode 100644 index 00000000..f380ab9b --- /dev/null +++ b/test/azure/run.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" +. ../util.sh +current_test=$(basename $(pwd)) + +docker-compose up -d +sleep 5 + +# A symlink for a known file in the volume is created so the test can check +# whether symlinks are preserved on backup. +docker-compose exec backup backup + +sleep 5 + +expect_running_containers "3" + +docker-compose run --rm az_cli \ + az storage blob download -f /dump/test.tar.gz -c test-container -n test.tar.gz +tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db + +pass "Found relevant files in untared remote backups." + +# The second part of this test checks if backups get deleted when the retention +# is set to 0 days (which it should not as it would mean all backups get deleted) +# TODO: find out if we can test actual deletion without having to wait for a day +BACKUP_RETENTION_DAYS="0" docker-compose up -d +sleep 5 + +docker-compose exec backup backup + +docker-compose run --rm az_cli \ + az storage blob download -f /dump/test.tar.gz -c test-container -n test.tar.gz +test -f ./local/test.tar.gz + +pass "Remote backups have not been deleted." + +docker-compose down --volumes From c4cfaa3dcbe28338d4d38121fa2c1f0c82babf37 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 24 Dec 2022 09:06:51 +0100 Subject: [PATCH 04/11] Implement pruning for Azure blob storage --- cmd/backup/config.go | 2 +- internal/storage/azure/azure.go | 75 +++++++++++++++++++++++++++++++-- test/azure/docker-compose.yml | 2 +- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 09ef3d21..398a96a1 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -66,7 +66,7 @@ type Config struct { AzureStorageAccountName string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"` AzureStorageContainerName string `split_words:"true"` - AzureStorageEndpoint string `split_words:"true" default:"https://%s.blob.core.windows.net/"` + AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 82165ceb..dd850bb5 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -4,14 +4,19 @@ package azure import ( + "bytes" "context" "fmt" "os" "path" + "sync" + "text/template" "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/offen/docker-volume-backup/internal/storage" + "github.com/offen/docker-volume-backup/internal/utilities" ) type azureBlobStorage struct { @@ -34,13 +39,26 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating shared Azure credential: %w", err) } - client, err := azblob.NewClientWithSharedKeyCredential(fmt.Sprintf(opts.Endpoint, opts.AccountName), cred, nil) + + endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err) + } + + var ep bytes.Buffer + if err := endpointTemplate.Execute(&ep, opts); err != nil { + return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err) + } + client, err := azblob.NewClientWithSharedKeyCredential(ep.String(), cred, nil) if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) } storage := azureBlobStorage{ client: client, containerName: opts.ContainerName, + StorageBackend: &storage.StorageBackend{ + Log: logFunc, + }, } return &storage, nil } @@ -57,7 +75,8 @@ func (b *azureBlobStorage) Copy(file string) error { return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err) } - _, err = b.client.UploadStream(context.TODO(), + _, err = b.client.UploadStream( + context.Background(), b.containerName, path.Base(file), fileReader, @@ -69,7 +88,55 @@ func (b *azureBlobStorage) Copy(file string) error { return nil } -// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend. +// Prune rotates away backups according to the configuration and provided +// deadline for the Azure Blob storage backend. func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { - return &storage.PruneStats{}, nil + pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{ + Prefix: &pruningPrefix, + }) + var matches []string + var totalCount uint + for pager.More() { + resp, err := pager.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err) + } + for _, v := range resp.Segment.BlobItems { + totalCount++ + if v.Properties.LastModified.Before(deadline) { + matches = append(matches, *v.Name) + } + } + } + + stats := storage.PruneStats{ + Total: totalCount, + Pruned: uint(len(matches)), + } + + if err := b.DoPrune(b.Name(), len(matches), int(totalCount), "Azure Blob Storage backup(s)", func() error { + wg := sync.WaitGroup{} + wg.Add(len(matches)) + var errors []error + + for _, match := range matches { + name := match + go func() { + _, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil) + if err != nil { + errors = append(errors, err) + } + wg.Done() + }() + } + wg.Wait() + if len(errors) != 0 { + return utilities.Join(errors...) + } + return nil + }); err != nil { + return &stats, err + } + + return &stats, nil } diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml index 5e3ff01b..a1146c33 100644 --- a/test/azure/docker-compose.yml +++ b/test/azure/docker-compose.yml @@ -34,7 +34,7 @@ services: AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1 AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== AZURE_STORAGE_CONTAINER_NAME: test-container - AZURE_STORAGE_ENDPOINT: http://storage:10000/%s/ + AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} From 92e6ccaacafec33152424462345c5404b18ae14d Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 24 Dec 2022 09:38:23 +0100 Subject: [PATCH 05/11] Add documentation for Azure Blob Storage --- README.md | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 32753136..a5253334 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Backup Docker volumes locally or to any S3 compatible storage. The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup. -It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__. +It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__. @@ -41,6 +41,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc - [Backing up to MinIO \(using Docker secrets\)](#backing-up-to-minio-using-docker-secrets) - [Backing up to WebDAV](#backing-up-to-webdav) - [Backing up to SSH](#backing-up-to-ssh) + - [Backing up to Azure Blob Storage](#backing-up-to-azure-blob-storage) - [Backing up locally](#backing-up-locally) - [Backing up to AWS S3 as well as locally](#backing-up-to-aws-s3-as-well-as-locally) - [Running on a custom cron schedule](#running-on-a-custom-cron-schedule) @@ -304,6 +305,23 @@ You can populate below template according to your requirements and use it as you # SSH_IDENTITY_PASSPHRASE="pass" +# The credential's account name when using Azure Blob Storage. + +# AZURE_STORAGE_ACCOUNT_NAME="account-name" + +# The credential's primary account key when using Azure Blob Storage. + +# AZURE_STORAGE_PRIMARY_ACCOUNT_KEY="" + +# The container name when using Azure Blob Storage. + +# AZURE_STORAGE_CONTAINER_NAME="container-name" + +# The service endpoint when using Azure Blob Storage. This is a template that +# will be passed the account name as shown in the value below. + +# AZURE_STORAGE_ENDPOINT="https://{{ .AccountName }}.blob.core.windows.net/" + # In addition to storing backups remotely, you can also keep local copies. # Pass a container-local path to store your backups if needed. You also need to # mount a local folder or Docker volume into that location (`/archive` @@ -1080,6 +1098,27 @@ volumes: data: ``` +### Backing up to Azure Blob Storage + +```yml +version: '3' + +services: + # ... define other services using the `data` volume here + backup: + image: offen/docker-volume-backup:v2 + environment: + AZURE_STORAGE_CONTAINER_NAME: backup-container + AZURE_STORAGE_ACCOUNT_NAME: account-name + AZURE_STORAGE_PRIMARY_ACCOUNT_NAME: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + volumes: + - data:/backup/my-app-backup:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + +volumes: + data: +``` + ### Backing up locally ```yml From 6e93c80aebae5a110eef85f2dd3de14aed19b1bf Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 24 Dec 2022 10:30:12 +0100 Subject: [PATCH 06/11] Add support for remote path --- cmd/backup/config.go | 1 + cmd/backup/script.go | 1 + internal/storage/azure/azure.go | 12 +++++++----- test/azure/docker-compose.yml | 1 + test/azure/run.sh | 4 ++-- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 398a96a1..88ebba1f 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -66,6 +66,7 @@ type Config struct { AzureStorageAccountName string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"` AzureStorageContainerName string `split_words:"true"` + AzureStoragePath string `split_words:"true"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` } diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 379df288..c4d34acb 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -197,6 +197,7 @@ func newScript() (*script, error) { AccountName: s.c.AzureStorageAccountName, PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey, Endpoint: s.c.AzureStorageEndpoint, + RemotePath: s.c.AzureStoragePath, } azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) if err != nil { diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index dd850bb5..8f306b30 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -8,7 +8,7 @@ import ( "context" "fmt" "os" - "path" + "path/filepath" "sync" "text/template" "time" @@ -31,6 +31,7 @@ type Config struct { ContainerName string PrimaryAccountKey string Endpoint string + RemotePath string } // NewStorageBackend creates and initializes a new Azure Blob Storage backend. @@ -57,7 +58,8 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error client: client, containerName: opts.ContainerName, StorageBackend: &storage.StorageBackend{ - Log: logFunc, + DestinationPath: opts.RemotePath, + Log: logFunc, }, } return &storage, nil @@ -74,11 +76,10 @@ func (b *azureBlobStorage) Copy(file string) error { if err != nil { return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err) } - _, err = b.client.UploadStream( context.Background(), b.containerName, - path.Base(file), + filepath.Join(b.DestinationPath, filepath.Base(file)), fileReader, nil, ) @@ -91,8 +92,9 @@ func (b *azureBlobStorage) Copy(file string) error { // Prune rotates away backups according to the configuration and provided // deadline for the Azure Blob storage backend. func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { + lookupPrefix := filepath.Join(b.DestinationPath, pruningPrefix) pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{ - Prefix: &pruningPrefix, + Prefix: &lookupPrefix, }) var matches []string var totalCount uint diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml index a1146c33..d2bceb8d 100644 --- a/test/azure/docker-compose.yml +++ b/test/azure/docker-compose.yml @@ -35,6 +35,7 @@ services: AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== AZURE_STORAGE_CONTAINER_NAME: test-container AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ + AZURE_STORAGE_PATH: 'path/to/backup' BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} diff --git a/test/azure/run.sh b/test/azure/run.sh index f380ab9b..00777068 100644 --- a/test/azure/run.sh +++ b/test/azure/run.sh @@ -18,7 +18,7 @@ sleep 5 expect_running_containers "3" docker-compose run --rm az_cli \ - az storage blob download -f /dump/test.tar.gz -c test-container -n test.tar.gz + az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db pass "Found relevant files in untared remote backups." @@ -32,7 +32,7 @@ sleep 5 docker-compose exec backup backup docker-compose run --rm az_cli \ - az storage blob download -f /dump/test.tar.gz -c test-container -n test.tar.gz + az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz test -f ./local/test.tar.gz pass "Remote backups have not been deleted." From c8b64e037c956e59001bb39d98e9d5c4c180e149 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 24 Dec 2022 10:35:36 +0100 Subject: [PATCH 07/11] Add azure to notifications doc --- README.md | 2 +- docs/NOTIFICATION-TEMPLATES.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a5253334..4f7ca1ac 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ You can populate below template according to your requirements and use it as you # AZURE_STORAGE_CONTAINER_NAME="container-name" # The service endpoint when using Azure Blob Storage. This is a template that -# will be passed the account name as shown in the value below. +# will be passed the account name as shown in the default value below. # AZURE_STORAGE_ENDPOINT="https://{{ .AccountName }}.blob.core.windows.net/" diff --git a/docs/NOTIFICATION-TEMPLATES.md b/docs/NOTIFICATION-TEMPLATES.md index 8af37af0..8bd5d605 100644 --- a/docs/NOTIFICATION-TEMPLATES.md +++ b/docs/NOTIFICATION-TEMPLATES.md @@ -25,7 +25,7 @@ Here is a list of all data passed to the template: * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`) * `Size`: size in bytes of the backup file * `Storages`: object that holds stats about each storage - * `Local`, `S3`, `WebDAV` or `SSH`: + * `Local`, `S3`, `WebDAV`, `Azure` or `SSH`: * `Total`: total number of backup files * `Pruned`: number of backup files that were deleted due to pruning rule * `PruneErrors`: number of backup files that were unable to be pruned From 245cb686ccfd375233935edd1e841148c9aa5869 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sat, 24 Dec 2022 10:41:11 +0100 Subject: [PATCH 08/11] Tidy go.mod file --- go.mod | 2 +- go.sum | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3bd4ca12..bd4a9980 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/offen/docker-volume-backup go 1.19 require ( + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 github.com/containrrr/shoutrrr v0.5.2 github.com/cosiner/argv v0.1.0 github.com/docker/docker v20.10.11+incompatible @@ -21,7 +22,6 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/containerd/containerd v1.6.6 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect diff --git a/go.sum b/go.sum index 5c9318f6..32bad400 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -54,6 +56,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo= @@ -100,6 +103,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -180,6 +184,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs= github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -250,6 +255,7 @@ github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -302,7 +308,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8= From af46b6139b7805d047d969dcb6e45719761c7d99 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sun, 25 Dec 2022 09:58:01 +0100 Subject: [PATCH 09/11] Allow use of managed identity credential --- README.md | 5 +++-- go.mod | 5 +++++ go.sum | 8 ++++++++ internal/storage/azure/azure.go | 32 +++++++++++++++++++++++--------- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4f7ca1ac..090f12ad 100644 --- a/README.md +++ b/README.md @@ -309,7 +309,8 @@ You can populate below template according to your requirements and use it as you # AZURE_STORAGE_ACCOUNT_NAME="account-name" -# The credential's primary account key when using Azure Blob Storage. +# The credential's primary account key when using Azure Blob Storage. If this +# is not given, the command tries to fall back to using a managed identity. # AZURE_STORAGE_PRIMARY_ACCOUNT_KEY="" @@ -1110,7 +1111,7 @@ services: environment: AZURE_STORAGE_CONTAINER_NAME: backup-container AZURE_STORAGE_ACCOUNT_NAME: account-name - AZURE_STORAGE_PRIMARY_ACCOUNT_NAME: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== volumes: - data:/backup/my-app-backup:ro - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/go.mod b/go.mod index bd4a9980..a4fa0121 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,9 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/containerd/containerd v1.6.6 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect @@ -31,6 +33,7 @@ require ( github.com/fatih/color v1.10.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.7.3 // indirect @@ -39,6 +42,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.1 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -53,6 +57,7 @@ require ( github.com/onsi/gomega v1.10.3 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rs/xid v1.4.0 // indirect golang.org/x/net v0.2.0 // indirect diff --git a/go.sum b/go.sum index 32bad400..91bd9e6c 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= @@ -10,6 +12,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjN github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -104,6 +108,8 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -185,6 +191,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs= github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -256,6 +263,7 @@ github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 8f306b30..2f09a27e 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -13,6 +13,7 @@ import ( "text/template" "time" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/offen/docker-volume-backup/internal/storage" @@ -36,24 +37,37 @@ type Config struct { // NewStorageBackend creates and initializes a new Azure Blob Storage backend. func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { - cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey) - if err != nil { - return nil, fmt.Errorf("NewStorageBackend: error creating shared Azure credential: %w", err) - } - endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) if err != nil { return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err) } - var ep bytes.Buffer if err := endpointTemplate.Execute(&ep, opts); err != nil { return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err) } - client, err := azblob.NewClientWithSharedKeyCredential(ep.String(), cred, nil) - if err != nil { - return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) + + var client *azblob.Client + if opts.PrimaryAccountKey != "" { + cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating shared key Azure credential: %w", err) + } + + client, err = azblob.NewClientWithSharedKeyCredential(ep.String(), cred, nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) + } + } else { + cred, err := azidentity.NewManagedIdentityCredential(nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating managed identity credential: %w", err) + } + client, err = azblob.NewClient(ep.String(), cred, nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) + } } + storage := azureBlobStorage{ client: client, containerName: opts.ContainerName, From f25db0d7a96ff6a9102ae03e447ceeb6c9a97924 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 26 Dec 2022 09:15:06 +0100 Subject: [PATCH 10/11] Use volume in tests --- test/azure/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml index d2bceb8d..34eab222 100644 --- a/test/azure/docker-compose.yml +++ b/test/azure/docker-compose.yml @@ -4,7 +4,7 @@ services: storage: image: mcr.microsoft.com/azure-storage/azurite volumes: - - ./foo:/data + - azurite_backup_data:/data command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data healthcheck: test: nc 127.0.0.1 10000 -z From 3c66274dcfe108522504cd274f9f73998b8c8de6 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 4 Jan 2023 20:30:14 +0100 Subject: [PATCH 11/11] Auto append trailing slash to endpoint if needed, clarify docs, tidy mod file --- README.md | 5 +++-- go.mod | 2 +- go.sum | 3 --- internal/storage/azure/azure.go | 6 ++++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 090f12ad..a3540eb7 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,8 @@ You can populate below template according to your requirements and use it as you # SSH_IDENTITY_PASSPHRASE="pass" -# The credential's account name when using Azure Blob Storage. +# The credential's account name when using Azure Blob Storage. This has to be +# set when using Azure Blob Storage. # AZURE_STORAGE_ACCOUNT_NAME="account-name" @@ -319,7 +320,7 @@ You can populate below template according to your requirements and use it as you # AZURE_STORAGE_CONTAINER_NAME="container-name" # The service endpoint when using Azure Blob Storage. This is a template that -# will be passed the account name as shown in the default value below. +# can be passed the account name as shown in the default value below. # AZURE_STORAGE_ENDPOINT="https://{{ .AccountName }}.blob.core.windows.net/" diff --git a/go.mod b/go.mod index a4fa0121..b852f765 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/offen/docker-volume-backup go 1.19 require ( + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 github.com/containrrr/shoutrrr v0.5.2 github.com/cosiner/argv v0.1.0 @@ -21,7 +22,6 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect diff --git a/go.sum b/go.sum index 91bd9e6c..30368f78 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= @@ -11,7 +10,6 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQO github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -107,7 +105,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 2f09a27e..c6c1d80b 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "text/template" "time" @@ -45,6 +46,7 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error if err := endpointTemplate.Execute(&ep, opts); err != nil { return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err) } + normalizedEndpoint := fmt.Sprintf("%s/", strings.TrimSuffix(ep.String(), "/")) var client *azblob.Client if opts.PrimaryAccountKey != "" { @@ -53,7 +55,7 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error return nil, fmt.Errorf("NewStorageBackend: error creating shared key Azure credential: %w", err) } - client, err = azblob.NewClientWithSharedKeyCredential(ep.String(), cred, nil) + client, err = azblob.NewClientWithSharedKeyCredential(normalizedEndpoint, cred, nil) if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) } @@ -62,7 +64,7 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating managed identity credential: %w", err) } - client, err = azblob.NewClient(ep.String(), cred, nil) + client, err = azblob.NewClient(normalizedEndpoint, cred, nil) if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) }