Add support AWS S3 providers

config.yaml

@@ -17,6 +17,7 @@ forms:
   url: forms:forms@tcp(127.0.0.1:5441)/forms
 
 storage:
+  backend: "do"
   endpoint: https://fra1.digitaloceanspaces.com
   bucket: bucket
   presigned_url_expiration: 3m

go.mod

@@ -25,11 +25,14 @@ require (
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
+	github.com/Workiva/go-datastructures v1.0.53 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect
+	github.com/aws/amazon-ssm-agent v0.0.0-20240627200756-18ab50687adf // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.0.2 // indirect
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect

go.sum

@@ -1168,6 +1168,8 @@ github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9
 github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
 github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40=
 github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
+github.com/Workiva/go-datastructures v1.0.53 h1:J6Y/52yX10Xc5JjXmGtWoSSxs3mZnGSaq37xZZh7Yig=
+github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
@@ -1201,6 +1203,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0=
 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/aws/amazon-ssm-agent v0.0.0-20240627200756-18ab50687adf h1:dl9VPwX1Xe3IhBp/8J+cy5YXx7xCEE3hJcNZ5URIJhk=
+github.com/aws/amazon-ssm-agent v0.0.0-20240627200756-18ab50687adf/go.mod h1:lsubMeeI5yNTh4sSjAXWrHHRoX6yrzjBrg6wV25dPkU=
 github.com/aws/aws-sdk-go v1.54.20 h1:FZ2UcXya7bUkvkpf7TaPmiL7EubK0go1nlXGLRwEsoo=
 github.com/aws/aws-sdk-go v1.54.20/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo=
@@ -1232,6 +1236,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
+github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=
+github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
@@ -1902,6 +1908,7 @@ github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdU
 github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY=
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
@@ -2074,6 +2081,7 @@ github.com/tdewolff/minify/v2 v2.12.4/go.mod h1:h+SRvSIX3kwgwTFOpSckvSxgax3uy8kZ
 github.com/tdewolff/parse/v2 v2.6.4/go.mod h1:woz0cgbLwFdtbjJu8PIKxhW05KplTFQkOdX78o+Jgrs=
 github.com/tdewolff/test v1.0.7/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
 github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
@@ -2082,6 +2090,7 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF
 github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs=
 github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=
@@ -2655,6 +2664,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=

internal/service/requests/submit_form.go

@@ -6,7 +6,7 @@ import (
 	"regexp"
 
 	validation "github.com/go-ozzo/ozzo-validation/v4"
-	"github.com/rarimo/geo-forms-svc/internal/storage"
+	"github.com/go-ozzo/ozzo-validation/v4/is"
 	"github.com/rarimo/geo-forms-svc/resources"
 )
 
@@ -33,7 +33,7 @@ func NewSubmitForm(r *http.Request) (req resources.SubmitFormRequest, err error)
 		"data/attributes/postal":   validation.Validate(req.Data.Attributes.Postal, validation.Required),
 		"data/attributes/phone":    validation.Validate(req.Data.Attributes.Phone, validation.Required),
 		"data/attributes/email":    validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(emailRegexp)),
-		"data/attributes/image":    validation.Validate(req.Data.Attributes.Image, validation.Required, validation.Match(storage.DOSpacesURLRegexp)),
+		"data/attributes/image":    validation.Validate(req.Data.Attributes.Image, validation.Required, is.URL),
 	}
 
 	return req, errs.Filter()

internal/storage/config.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -29,13 +30,13 @@ type storager struct {
 	getter kv.Getter
 }
 
-// Storage works only with DigitalOceanSpaces.
+// Storage support DigitalOceanSpaces and AWS S3
 // Other providers are not supported.
 func (c *storager) Storage() *Storage {
 	return c.once.Do(func() interface{} {
 		var envCfg struct {
-			SpacesKey    string `dig:"SPACES_KEY,clear"`
-			SpacesSecret string `dig:"SPACES_SECRET,clear"`
+			S3Key    string `dig:"S3_KEY,clear"`
+			S3Secret string `dig:"S3_SECRET,clear"`
 		}
 
 		err := dig.Out(&envCfg).Now()
@@ -44,9 +45,11 @@ func (c *storager) Storage() *Storage {
 		}
 
 		var cfg struct {
+			Backend                string         `fig:"backend,required"`
 			Endpoint               string         `fig:"endpoint,required"`
 			Bucket                 string         `fig:"bucket,required"`
 			PresignedURLExpiration *time.Duration `fig:"presigned_url_expiration"`
+			Region                 *string        `fig:"region"`
 		}
 
 		err = figure.Out(&cfg).
@@ -56,14 +59,24 @@ func (c *storager) Storage() *Storage {
 			panic(fmt.Errorf("failed to figure out s3 storage config: %w", err))
 		}
 
+		switch cfg.Backend {
+		case digitalOceanBackend, awsBackend:
+		default:
+			panic(errors.New("invalid backend provided"))
+		}
+
 		if cfg.PresignedURLExpiration == nil {
 			cfg.PresignedURLExpiration = &defaultPresignedURLExpiration
 		}
 
+		if cfg.Region == nil {
+			cfg.Region = aws.String(defaultRegion)
+		}
+
 		s3Config := &aws.Config{
-			Credentials:      credentials.NewStaticCredentials(envCfg.SpacesKey, envCfg.SpacesSecret, ""),
+			Credentials:      credentials.NewStaticCredentials(envCfg.S3Key, envCfg.S3Secret, ""),
 			Endpoint:         aws.String(cfg.Endpoint),
-			Region:           aws.String("us-east-1"),
+			Region:           cfg.Region,
 			S3ForcePathStyle: aws.Bool(false),
 		}
 
@@ -78,6 +91,8 @@ func (c *storager) Storage() *Storage {
 			client:                 s3Client,
 			bucket:                 cfg.Bucket,
 			presignedURLExpiration: *cfg.PresignedURLExpiration,
+			backend:                cfg.Backend,
+			region:                 *cfg.Region,
 		}
 	}).(*Storage)
 }

internal/storage/main.go

@@ -6,21 +6,22 @@ import (
 	"io"
 	"net/url"
 
+	"github.com/aws/amazon-ssm-agent/agent/s3util"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
 func (s *Storage) GetImageBase64(object *url.URL) (*string, error) {
-	spacesURL, err := parseDOSpacesURL(object)
+	bucket, key, err := s.bucketAndKey(object)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse url [%s]: %w", object.String(), err)
+		return nil, fmt.Errorf("failed to get bucket and key: %w", err)
 	}
 
 	output, err := s.client.GetObject(&s3.GetObjectInput{
-		Bucket: aws.String(spacesURL.Bucket),
-		Key:    aws.String(spacesURL.Key),
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to get object meta: %w", err)
@@ -38,23 +39,23 @@ func (s *Storage) GetImageBase64(object *url.URL) (*string, error) {
 }
 
 func (s *Storage) ValidateImage(object *url.URL, id string) error {
-	spacesURL, err := parseDOSpacesURL(object)
+	bucket, key, err := s.bucketAndKey(object)
 	if err != nil {
-		return fmt.Errorf("failed to parse url [%s]: %w", object.String(), err)
+		return fmt.Errorf("failed to get bucket and key: %w", err)
 	}
 
-	if spacesURL.Bucket != s.bucket {
+	if bucket != s.bucket {
 		return ErrInvalidBucket
 	}
 
-	if spacesURL.Key != id {
+	if key != id {
 		return ErrInvalidKey
 	}
 
 	// output can't be nil
 	output, err := s.client.HeadObject(&s3.HeadObjectInput{
-		Bucket: aws.String(spacesURL.Bucket),
-		Key:    aws.String(spacesURL.Key),
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
 	})
 	if err != nil {
 		return fmt.Errorf("failed to get image meta: %w", err)
@@ -71,6 +72,26 @@ func (s *Storage) ValidateImage(object *url.URL, id string) error {
 	return nil
 }
 
+func (s *Storage) bucketAndKey(link *url.URL) (bucket, key string, err error) {
+	switch s.backend {
+	case digitalOceanBackend:
+		spacesURL, err := parseDOSpacesURL(link)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to parse url [%s]: %w", link, err)
+		}
+		return spacesURL.Bucket, spacesURL.Key, nil
+	case awsBackend:
+		s3URL := s3util.ParseAmazonS3URL(nil, link)
+		if s3URL.Region != s.region {
+			return "", "", ErrRegionMismatched
+		}
+		return s3URL.Bucket, s3URL.Key, nil
+		// should be never happened
+	default:
+		return "", "", errors.New("invalid backend")
+	}
+}
+
 func (s *Storage) GeneratePutURL(fileName, contentType string, contentLength int64) (signedURL, key string, err error) {
 	key = uuid.New().String()
 	if fileName != "" {
@@ -96,7 +117,7 @@ func parseDOSpacesURL(object *url.URL) (*SpacesURL, error) {
 		URL: object,
 	}
 
-	components := DOSpacesURLRegexp.FindStringSubmatch(object.String())
+	components := doSpacesURLRegexp.FindStringSubmatch(object.String())
 	if components == nil {
 		return nil, ErrURLRegexp
 	}

internal/storage/types.go

@@ -9,16 +9,22 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`)
+var doSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`)
 
-const maxImageSize = 1 << 22 // 4mb
+const (
+	maxImageSize        = 1 << 22 // 4mb
+	defaultRegion       = "us-east-1"
+	digitalOceanBackend = "do"
+	awsBackend          = "aws"
+)
 
 var (
 	ErrImageTooLarge      = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize)
 	ErrIncorrectImageType = fmt.Errorf("incorrect object type, must be image/png or image/jpeg")
 	ErrURLRegexp          = fmt.Errorf("url don't match regexp")
 	ErrInvalidBucket      = fmt.Errorf("invalid bucket")
 	ErrInvalidKey         = fmt.Errorf("invalid key")
+	ErrRegionMismatched   = fmt.Errorf("aws datacenter region mismatched")
 
 	defaultPresignedURLExpiration = 5 * time.Minute
 )
@@ -27,6 +33,8 @@ type Storage struct {
 	client                 *s3.S3
 	bucket                 string
 	presignedURLExpiration time.Duration
+	backend                string
+	region                 string
 }
 
 type SpacesURL struct {