Merge pull request #4 from rarimo/feat/s3-integrations

Add endpoint for getting url for put image in s3 storage

config.yaml

@@ -18,8 +18,8 @@ forms:
 
 storage:
   endpoint: https://fra1.digitaloceanspaces.com
-  allowed_buckets:
-    - bucket
+  bucket: bucket
+  presigned_url_expiration: 3m
 
 auth:
   addr: http://127.0.0.1:5000

docs/spec/components/schemas/Form.yaml

@@ -66,7 +66,8 @@ allOf:
             type: string
             example: "+13165282105"
           email:
-            type: [email protected]
+            type: string
+            example: [email protected]
           image:
             type: string
             description: |

docs/spec/components/schemas/UploadImage.yaml

@@ -0,0 +1,23 @@
+allOf:
+  - $ref: '#/components/schemas/UploadImageKey'
+  - type: object
+    x-go-is-request: true
+    required:
+      - attributes
+    properties:
+      attributes:
+        type: object
+        required:
+          - content_type
+          - content_length
+        properties:
+          content_type:
+            type: string
+            example: image/png
+            description: Allowed content-type is `image/png` or `image/jpeg`
+          content_length:
+            type: integer
+            format: int64
+            example: 150000
+            description: Image size. It cannot be more than 4 megabytes.
+

docs/spec/components/schemas/UploadImageKey.yaml

@@ -0,0 +1,7 @@
+type: object
+required:
+  - type
+properties:
+  type:
+    type: string
+    enum: [ upload_image ]

docs/spec/components/schemas/UploadImageResponse.yaml

@@ -0,0 +1,15 @@
+allOf:
+  - $ref: '#/components/schemas/UploadImageResponseKey'
+  - type: object
+    required:
+      - attributes
+    properties:
+      attributes:
+        type: object
+        required:
+          - url
+        properties:
+          url:
+            type: string
+            description: Pre-signed URL to upload the file
+            example: https://bucket.nyc3.digitaloceanspaces.com/somefile?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=DO00PTJRCBZELX6E4EEK%2F20240722%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240722T133921Z&X-Amz-Expires=300&X-Amz-SignedHeaders=host&X-Amz-Signature=940c9058b90e8836b03470fdb51af1f24baabc16a7a83b80352d3d618aa4f23f

docs/spec/components/schemas/UploadImageResponseKey.yaml

@@ -0,0 +1,11 @@
+type: object
+required:
+  - id
+  - type
+properties:
+  id:
+    type: string
+    description: UUID to check form submission status
+  type:
+    type: string
+    enum: [ upload_image_response ]

docs/spec/paths/integrations@forms-svc@v1@image.yaml

@@ -0,0 +1,44 @@
+post:
+  tags:
+    - User form
+  summary: Generate pre-signed url
+  description: |
+    Generate pre-signed URL for the provided content-length 
+    and content-type, with a configurable lifetime. 
+    The response contains a URL with a signature and 
+    other information that should be used to upload image 
+    in S3 Storage. The name is generated on the server side.
+    'verified: true' must be specified in the JWT.
+    The cooldown of this endpoint is the same as in the submit form.
+  operationId: uploadImage
+  security:
+    - BearerAuth: []
+  requestBody:
+    content:
+      application/vnd.api+json:
+        schema:
+          type: object
+          required:
+            - data
+          properties:
+            data:
+              $ref: '#/components/schemas/UploadImage'
+  responses:
+    200:
+      description: "Success"
+      content:
+        application/vnd.api+json:
+          schema:
+            $ref: '#/components/schemas/UploadImageResponse'
+    400:
+      $ref: '#/components/responses/invalidParameter'
+    401:
+      $ref: '#/components/responses/invalidAuth'
+    429:
+      description: "It is necessary to wait some time before sending the next form"
+      content:
+        application/vnd.api+json:
+          schema:
+            $ref: '#/components/schemas/Errors'
+    500:
+      $ref: '#/components/responses/internalError'

go.mod

@@ -9,10 +9,13 @@ require (
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/uuid v1.6.0
+	github.com/pkg/errors v0.9.1
 	github.com/rarimo/geo-auth-svc v0.1.0
 	github.com/rubenv/sql-migrate v1.6.1
 	gitlab.com/distributed_lab/ape v1.7.1
 	gitlab.com/distributed_lab/dig v0.0.0-20230207152643-c44f80a4294c
+	gitlab.com/distributed_lab/figure v2.1.2+incompatible
 	gitlab.com/distributed_lab/figure/v3 v3.1.4
 	gitlab.com/distributed_lab/kit v1.11.3
 	gitlab.com/distributed_lab/logan v3.8.1+incompatible
@@ -59,7 +62,6 @@ require (
 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/rs/cors v1.8.3 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -77,7 +79,6 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
-	gitlab.com/distributed_lab/figure v2.1.2+incompatible // indirect
 	gitlab.com/distributed_lab/lorem v0.2.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/crypto v0.24.0 // indirect

internal/service/handlers/legacy_submit_form.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rarimo/geo-auth-svc/pkg/auth"
 	"github.com/rarimo/geo-forms-svc/internal/data"
 	"github.com/rarimo/geo-forms-svc/internal/service/requests"
 	"github.com/rarimo/geo-forms-svc/resources"
@@ -20,6 +21,10 @@ func LegacySubmitForm(w http.ResponseWriter, r *http.Request) {
 	}
 
 	nullifier := strings.ToLower(UserClaims(r)[0].Nullifier)
+	if !auth.Authenticates(UserClaims(r), auth.VerifiedGrant(nullifier)) {
+		ape.RenderErr(w, problems.Unauthorized())
+		return
+	}
 
 	formStatus, err := FormsQ(r).Last(nullifier)
 	if err != nil {

internal/service/handlers/submit_form.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/rarimo/geo-auth-svc/pkg/auth"
 	"github.com/rarimo/geo-forms-svc/internal/data"
 	"github.com/rarimo/geo-forms-svc/internal/service/requests"
 	"github.com/rarimo/geo-forms-svc/internal/storage"
@@ -23,6 +24,10 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) {
 	}
 
 	nullifier := strings.ToLower(UserClaims(r)[0].Nullifier)
+	if !auth.Authenticates(UserClaims(r), auth.VerifiedGrant(nullifier)) {
+		ape.RenderErr(w, problems.Unauthorized())
+		return
+	}
 
 	lastForm, err := FormsQ(r).Last(nullifier)
 	if err != nil {

internal/service/handlers/upload_image.go

@@ -0,0 +1,62 @@
+package handlers
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/rarimo/geo-auth-svc/pkg/auth"
+	"github.com/rarimo/geo-forms-svc/internal/service/requests"
+	"github.com/rarimo/geo-forms-svc/resources"
+	"gitlab.com/distributed_lab/ape"
+	"gitlab.com/distributed_lab/ape/problems"
+)
+
+func UploadImage(w http.ResponseWriter, r *http.Request) {
+	req, err := requests.NewUploadImage(r)
+	if err != nil {
+		ape.RenderErr(w, problems.BadRequest(err)...)
+		return
+	}
+
+	nullifier := strings.ToLower(UserClaims(r)[0].Nullifier)
+	if !auth.Authenticates(UserClaims(r), auth.VerifiedGrant(nullifier)) {
+		ape.RenderErr(w, problems.Unauthorized())
+		return
+	}
+
+	lastForm, err := FormsQ(r).Last(nullifier)
+	if err != nil {
+		Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier)
+		ape.RenderErr(w, problems.InternalError())
+		return
+	}
+
+	if lastForm != nil {
+		next := lastForm.CreatedAt.Add(Forms(r).Cooldown)
+		if next.After(time.Now().UTC()) {
+			Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next)
+			ape.RenderErr(w, problems.TooManyRequests())
+			return
+		}
+	}
+
+	signedURL, key, err := Storage(r).GeneratePutURL(req.Data.Attributes.ContentType, req.Data.Attributes.ContentLength)
+	if err != nil {
+		Log(r).WithError(err).Error("Failed to generate pre-signed url")
+		ape.RenderErr(w, problems.InternalError())
+		return
+	}
+
+	ape.Render(w, resources.UploadImageResponseResponse{
+		Data: resources.UploadImageResponse{
+			Key: resources.Key{
+				ID:   key,
+				Type: resources.UPLOAD_IMAGE_RESPONSE,
+			},
+			Attributes: resources.UploadImageResponseAttributes{
+				Url: signedURL,
+			},
+		},
+	})
+}

internal/service/requests/legacy_submit_form.go

@@ -12,7 +12,7 @@ import (
 )
 
 // 4 b64 letters encode 3 bytes, max image size = 12 MB -> (12/3)*4 * (1 << 20)
-const maxImageSize = (1 << 20) * 16
+const maxBase64ImageSize = (1 << 20) * 16
 
 func NewLegacySubmitForm(r *http.Request) (req resources.SubmitFormRequest, err error) {
 	if err = json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -35,7 +35,7 @@ func NewLegacySubmitForm(r *http.Request) (req resources.SubmitFormRequest, err
 		"data/attributes/postal":   validation.Validate(req.Data.Attributes.Postal, validation.Required),
 		"data/attributes/phone":    validation.Validate(req.Data.Attributes.Phone, validation.Required),
 		"data/attributes/email":    validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`))),
-		"data/attributes/image":    validation.Validate(req.Data.Attributes.Image, validation.Required, is.Base64, validation.Length(0, maxImageSize)),
+		"data/attributes/image":    validation.Validate(req.Data.Attributes.Image, validation.Required, is.Base64, validation.Length(0, maxBase64ImageSize)),
 	}
 
 	return req, errs.Filter()

internal/service/requests/upload_image.go

@@ -0,0 +1,26 @@
+package requests
+
+import (
+	"encoding/json"
+	"net/http"
+
+	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/rarimo/geo-forms-svc/resources"
+)
+
+const maxImageSize = 1 << 22
+
+func NewUploadImage(r *http.Request) (req resources.UploadImageRequest, err error) {
+	if err = json.NewDecoder(r.Body).Decode(&req); err != nil {
+		err = newDecodeError("body", err)
+		return
+	}
+
+	errs := validation.Errors{
+		"data/type":                      validation.Validate(req.Data.Type, validation.Required, validation.In(resources.UPLOAD_IMAGE)),
+		"data/attributes/content_type":   validation.Validate(req.Data.Attributes.ContentType, validation.Required, validation.In("image/png", "image/jpeg")),
+		"data/attributes/content_length": validation.Validate(req.Data.Attributes.ContentLength, validation.Required, validation.Length(1, int(maxImageSize))),
+	}
+
+	return req, errs.Filter()
+}

internal/service/router.go

@@ -25,6 +25,7 @@ func Run(ctx context.Context, cfg config.Config) {
 	)
 	r.Route("/integrations/geo-forms-svc/v1", func(r chi.Router) {
 		r.Use(handlers.AuthMiddleware(cfg.Auth(), cfg.Log()))
+		r.Post("/image", handlers.UploadImage)
 		r.Route("/status", func(r chi.Router) {
 			r.Get("/{id}", handlers.StatusByID)
 			r.Get("/last", handlers.LastStatus)

internal/storage/config.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -43,8 +44,9 @@ func (c *storager) Storage() *Storage {
 		}
 
 		var cfg struct {
-			Endpoint       string   `fig:"endpoint,required"`
-			AllowedBuckets []string `fig:"allowed_buckets,required"`
+			Endpoint               string         `fig:"endpoint,required"`
+			Bucket                 string         `fig:"bucket,required"`
+			PresignedURLExpiration *time.Duration `fig:"presigned_url_expiration"`
 		}
 
 		err = figure.Out(&cfg).
@@ -54,6 +56,10 @@ func (c *storager) Storage() *Storage {
 			panic(fmt.Errorf("failed to figure out s3 storage config: %w", err))
 		}
 
+		if cfg.PresignedURLExpiration == nil {
+			cfg.PresignedURLExpiration = &defaultPresignedURLExpiration
+		}
+
 		s3Config := &aws.Config{
 			Credentials:      credentials.NewStaticCredentials(envCfg.SpacesKey, envCfg.SpacesSecret, ""),
 			Endpoint:         aws.String(cfg.Endpoint),
@@ -69,8 +75,9 @@ func (c *storager) Storage() *Storage {
 		s3Client := s3.New(newSession)
 
 		return &Storage{
-			client:         s3Client,
-			allowedBuckets: cfg.AllowedBuckets,
+			client:                 s3Client,
+			bucket:                 cfg.Bucket,
+			presignedURLExpiration: *cfg.PresignedURLExpiration,
 		}
 	}).(*Storage)
 }