diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml index 78397e7..0a471f7 100644 --- a/.github/workflows/tag.yaml +++ b/.github/workflows/tag.yaml @@ -3,6 +3,7 @@ on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+' env: CI_JOB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/spec/components/schemas/FormStatus.yaml b/docs/spec/components/schemas/FormStatus.yaml index cbfd4b9..4e49699 100644 --- a/docs/spec/components/schemas/FormStatus.yaml +++ b/docs/spec/components/schemas/FormStatus.yaml @@ -14,8 +14,9 @@ allOf: properties: status: type: string - enum: [ accepted, processed ] + enum: [ created, accepted, processed ] description: | + Created - the empty form was created and now user can't use legacy submit Accepted - the data was saved by the service for further processing Processed - the data is processed and stored created_at: diff --git a/docs/spec/paths/integrations@forms-svc@v1@form.yaml b/docs/spec/paths/integrations@forms-svc@v1@form.yaml index 1fd89ca..80632f8 100644 --- a/docs/spec/paths/integrations@forms-svc@v1@form.yaml +++ b/docs/spec/paths/integrations@forms-svc@v1@form.yaml @@ -3,13 +3,11 @@ post: - User form summary: Submit lightweight user answers description: | - Send user answers and return their current status. + Sending the filled form. Requires created empty form + linked to the name of the file with the photo. + Return form current status. - The image is a link to s3 storage - - Only a user with a confirmed passport can send the form ('verified: true' in JWT). - There is a configurable delay before the same user - can submit another form. + The image is a link to s3 storage operationId: lightweightSubmitForm security: - BearerAuth: [] @@ -34,11 +32,13 @@ post: $ref: '#/components/responses/invalidParameter' 401: $ref: '#/components/responses/invalidAuth' - 429: - description: "It is necessary to wait some time before sending the next form" + 403: + description: "Empty form absent for user, but processed or accepted one exists" content: application/vnd.api+json: schema: $ref: '#/components/schemas/Errors' + 404: + $ref: '#/components/responses/notFound' 500: $ref: '#/components/responses/internalError' diff --git a/docs/spec/paths/integrations@forms-svc@v1@image.yaml b/docs/spec/paths/integrations@forms-svc@v1@image.yaml index 3bacb8f..2ee6196 100644 --- a/docs/spec/paths/integrations@forms-svc@v1@image.yaml +++ b/docs/spec/paths/integrations@forms-svc@v1@image.yaml @@ -10,6 +10,7 @@ post: in S3 Storage. The name is generated on the server side. 'verified: true' must be specified in the JWT. The cooldown of this endpoint is the same as in the submit form. + This endpoint creates an empty form that can be submitted on `/form` operationId: uploadImage security: - BearerAuth: [] diff --git a/internal/data/forms.go b/internal/data/forms.go index 58e6b24..995da25 100644 --- a/internal/data/forms.go +++ b/internal/data/forms.go @@ -6,10 +6,29 @@ import ( ) const ( + CreatedStatus = "created" AcceptedStatus = "accepted" ProcessedStatus = "processed" ) +const ( + ColStatus = "status" + ColName = "name" + ColSurname = "surname" + ColIDNum = "id_num" + ColBirthday = "birthday" + ColCitizen = "citizen" + ColVisited = "visited" + ColPurpose = "purpose" + ColCountry = "country" + ColCity = "city" + ColAddress = "address" + ColPostal = "postal" + ColPhone = "phone" + ColEmail = "email" + ColImageURL = "image_url" +) + type Form struct { ID string `db:"id"` Nullifier string `db:"nullifier"` @@ -45,7 +64,8 @@ type FormStatus struct { type FormsQ interface { New() FormsQ Insert(*Form) (*FormStatus, error) - Update(status string) error + + Update(map[string]interface{}) error Select() ([]*Form, error) Limit(uint64) FormsQ @@ -54,5 +74,5 @@ type FormsQ interface { Last(nullifier string) (*FormStatus, error) FilterByID(ids ...string) FormsQ - FilterByStatus(status string) FormsQ + FilterByStatus(status ...string) FormsQ } diff --git a/internal/data/pg/forms.go b/internal/data/pg/forms.go index 340cb50..d1999cd 100644 --- a/internal/data/pg/forms.go +++ b/internal/data/pg/forms.go @@ -18,6 +18,7 @@ const ( type formsQ struct { db *pgdb.DB selector squirrel.SelectBuilder + last squirrel.SelectBuilder updater squirrel.UpdateBuilder } @@ -25,6 +26,7 @@ func NewForms(db *pgdb.DB) data.FormsQ { return &formsQ{ db: db, selector: squirrel.Select("*").From(formsTable), + last: squirrel.Select(formsStatusFields).From(formsTable), updater: squirrel.Update(formsTable), } } @@ -36,7 +38,7 @@ func (q *formsQ) New() data.FormsQ { func (q *formsQ) Insert(form *data.Form) (*data.FormStatus, error) { var res data.FormStatus - stmt := squirrel.Insert(formsTable).SetMap(map[string]interface{}{ + values := map[string]interface{}{ "nullifier": form.Nullifier, "status": form.Status, "name": form.Name, @@ -54,7 +56,13 @@ func (q *formsQ) Insert(form *data.Form) (*data.FormStatus, error) { "email": form.Email, "image": form.Image, "image_url": form.ImageURL, - }).Suffix("RETURNING id, nullifier, status, created_at, updated_at") + } + + if form.ID != "" { + values["id"] = form.ID + } + + stmt := squirrel.Insert(formsTable).SetMap(values).Suffix("RETURNING id, nullifier, status, created_at, updated_at") if err := q.db.Get(&res, stmt); err != nil { return nil, fmt.Errorf("insert form: %w", err) @@ -63,8 +71,8 @@ func (q *formsQ) Insert(form *data.Form) (*data.FormStatus, error) { return &res, nil } -func (q *formsQ) Update(status string) error { - if err := q.db.Exec(q.updater.Set("status", status)); err != nil { +func (q *formsQ) Update(fields map[string]any) error { + if err := q.db.Exec(q.updater.SetMap(fields)); err != nil { return fmt.Errorf("update forms: %w", err) } @@ -98,7 +106,7 @@ func (q *formsQ) Get(id string) (*data.FormStatus, error) { func (q *formsQ) Last(nullifier string) (*data.FormStatus, error) { var res data.FormStatus - stmt := squirrel.Select(formsStatusFields).From(formsTable).Where(squirrel.Eq{"nullifier": nullifier}).OrderBy("created_at DESC") + stmt := q.last.Where(squirrel.Eq{"nullifier": nullifier}).OrderBy("created_at DESC") if err := q.db.Get(&res, stmt); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -122,12 +130,13 @@ func (q *formsQ) FilterByNullifier(nullifier string) data.FormsQ { return q.applyCondition(squirrel.Eq{"nullifier": nullifier}) } -func (q *formsQ) FilterByStatus(status string) data.FormsQ { +func (q *formsQ) FilterByStatus(status ...string) data.FormsQ { return q.applyCondition(squirrel.Eq{"status": status}) } func (q *formsQ) applyCondition(cond squirrel.Sqlizer) data.FormsQ { q.selector = q.selector.Where(cond) + q.last = q.selector.Where(cond) q.updater = q.updater.Where(cond) return q } diff --git a/internal/service/handlers/last_status.go b/internal/service/handlers/last_status.go index faa2105..05e7884 100644 --- a/internal/service/handlers/last_status.go +++ b/internal/service/handlers/last_status.go @@ -34,7 +34,7 @@ func LastStatus(w http.ResponseWriter, r *http.Request) { func newFormStatusResponse(formStatus *data.FormStatus) resources.FormStatusResponse { untilNextForm := time.Now().UTC().Unix() - formStatus.NextFormAt.Unix() - if untilNextForm < 0 { + if untilNextForm < 0 || formStatus.Status == data.CreatedStatus { untilNextForm = 0 } diff --git a/internal/service/handlers/status_by_id.go b/internal/service/handlers/status_by_id.go index 778d490..f30335d 100644 --- a/internal/service/handlers/status_by_id.go +++ b/internal/service/handlers/status_by_id.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/rarimo/geo-auth-svc/pkg/auth" + "github.com/rarimo/geo-forms-svc/internal/data" "github.com/rarimo/geo-forms-svc/internal/service/requests" "gitlab.com/distributed_lab/ape" "gitlab.com/distributed_lab/ape/problems" @@ -54,7 +55,11 @@ func StatusByID(w http.ResponseWriter, r *http.Request) { ape.RenderErr(w, problems.Unauthorized()) return } - formStatus.NextFormAt = lastStatus.CreatedAt.Add(Forms(r).Cooldown) + + formStatus.NextFormAt = lastStatus.CreatedAt + if lastStatus.Status != data.CreatedStatus { + formStatus.NextFormAt = lastStatus.CreatedAt.Add(Forms(r).Cooldown) + } ape.Render(w, newFormStatusResponse(formStatus)) } diff --git a/internal/service/handlers/submit_form.go b/internal/service/handlers/submit_form.go index 323f9c1..6bea2cd 100644 --- a/internal/service/handlers/submit_form.go +++ b/internal/service/handlers/submit_form.go @@ -5,7 +5,6 @@ import ( "net/http" "net/url" "strings" - "time" validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/rarimo/geo-auth-svc/pkg/auth" @@ -36,13 +35,15 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { return } - if lastForm != nil { - next := lastForm.CreatedAt.Add(Forms(r).Cooldown) - if next.After(time.Now().UTC()) { - Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next) - ape.RenderErr(w, problems.TooManyRequests()) - return - } + if lastForm == nil { + ape.RenderErr(w, problems.NotFound()) + return + } + + if lastForm.Status != data.CreatedStatus { + Log(r).Debugf("User last form must have %s status, got %s", data.CreatedStatus, lastForm.Status) + ape.RenderErr(w, problems.Forbidden()) + return } imageURL, err := url.Parse(req.Data.Attributes.Image) @@ -52,7 +53,7 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { return } - if err = Storage(r).ValidateImage(imageURL); err != nil { + if err = Storage(r).ValidateImage(imageURL, lastForm.ID); err != nil { if storage.IsBadRequestError(err) { ape.RenderErr(w, problems.BadRequest(validation.Errors{ "image": err, @@ -66,34 +67,37 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { } userData := req.Data.Attributes - form := &data.Form{ - Nullifier: nullifier, - Status: data.AcceptedStatus, - Name: userData.Name, - Surname: userData.Surname, - IDNum: userData.IdNum, - Birthday: userData.Birthday, - Citizen: userData.Citizen, - Visited: userData.Visited, - Purpose: userData.Purpose, - Country: userData.Country, - City: userData.City, - Address: userData.Address, - Postal: userData.Postal, - Phone: userData.Phone, - Email: userData.Email, - Image: nil, - ImageURL: sql.NullString{String: userData.Image, Valid: true}, + err = FormsQ(r).FilterByID(lastForm.ID).Update(map[string]interface{}{ + data.ColStatus: data.AcceptedStatus, + data.ColName: userData.Name, + data.ColSurname: userData.Surname, + data.ColIDNum: userData.IdNum, + data.ColBirthday: userData.Birthday, + data.ColCitizen: userData.Citizen, + data.ColVisited: userData.Visited, + data.ColPurpose: userData.Purpose, + data.ColCountry: userData.Country, + data.ColCity: userData.City, + data.ColAddress: userData.Address, + data.ColPostal: userData.Postal, + data.ColPhone: userData.Phone, + data.ColEmail: userData.Email, + data.ColImageURL: sql.NullString{String: userData.Image, Valid: true}, + }) + if err != nil { + Log(r).WithError(err).Error("failed to insert form") + ape.RenderErr(w, problems.InternalError()) + return } - formStatus, err := FormsQ(r).Insert(form) + lastForm, err = FormsQ(r).Last(nullifier) if err != nil { - Log(r).WithError(err).Error("failed to insert form") + Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier) ape.RenderErr(w, problems.InternalError()) return } - formStatus.NextFormAt = formStatus.CreatedAt.Add(Forms(r).Cooldown) + lastForm.NextFormAt = lastForm.CreatedAt.Add(Forms(r).Cooldown) - ape.Render(w, newFormStatusResponse(formStatus)) + ape.Render(w, newFormStatusResponse(lastForm)) } diff --git a/internal/service/handlers/upload_image.go b/internal/service/handlers/upload_image.go index 5250d60..6ee3e46 100644 --- a/internal/service/handlers/upload_image.go +++ b/internal/service/handlers/upload_image.go @@ -1,11 +1,13 @@ package handlers import ( + "fmt" "net/http" "strings" "time" "github.com/rarimo/geo-auth-svc/pkg/auth" + "github.com/rarimo/geo-forms-svc/internal/data" "github.com/rarimo/geo-forms-svc/internal/service/requests" "github.com/rarimo/geo-forms-svc/resources" "gitlab.com/distributed_lab/ape" @@ -32,31 +34,75 @@ func UploadImage(w http.ResponseWriter, r *http.Request) { return } - if lastForm != nil { - next := lastForm.CreatedAt.Add(Forms(r).Cooldown) - if next.After(time.Now().UTC()) { - Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next) - ape.RenderErr(w, problems.TooManyRequests()) + if lastForm == nil { + signedURL, id, err := newCreatedFormWithURL(r, nullifier, req.Data.Attributes.ContentType, req.Data.Attributes.ContentLength) + if err != nil { + Log(r).WithError(err).Error("Failed to create form") + ape.RenderErr(w, problems.InternalError()) return } + + ape.Render(w, newUploadImageResponse(id, signedURL)) + return + } + + if lastForm.Status == data.CreatedStatus { + signedURL, id, err := Storage(r).GeneratePutURL(lastForm.ID, req.Data.Attributes.ContentType, req.Data.Attributes.ContentLength) + if err != nil { + Log(r).WithError(err).Error("Failed to generate pre-signed url") + ape.RenderErr(w, problems.InternalError()) + return + } + + ape.Render(w, newUploadImageResponse(id, signedURL)) + return + } + + next := lastForm.CreatedAt.Add(Forms(r).Cooldown) + if next.After(time.Now().UTC()) { + Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next) + ape.RenderErr(w, problems.TooManyRequests()) + return } - signedURL, key, err := Storage(r).GeneratePutURL(req.Data.Attributes.ContentType, req.Data.Attributes.ContentLength) + signedURL, id, err := newCreatedFormWithURL(r, nullifier, req.Data.Attributes.ContentType, req.Data.Attributes.ContentLength) if err != nil { - Log(r).WithError(err).Error("Failed to generate pre-signed url") + Log(r).WithError(err).Error("Failed to create form") ape.RenderErr(w, problems.InternalError()) return } - ape.Render(w, resources.UploadImageResponseResponse{ + ape.Render(w, newUploadImageResponse(id, signedURL)) +} + +func newUploadImageResponse(id, signedURL string) resources.UploadImageResponseResponse { + return resources.UploadImageResponseResponse{ Data: resources.UploadImageResponse{ Key: resources.Key{ - ID: key, + ID: id, Type: resources.UPLOAD_IMAGE_RESPONSE, }, Attributes: resources.UploadImageResponseAttributes{ Url: signedURL, }, }, + } +} + +func newCreatedFormWithURL(r *http.Request, nullifier, contentType string, contentLength int64) (string, string, error) { + signedURL, id, err := Storage(r).GeneratePutURL("", contentType, contentLength) + if err != nil { + return "", "", fmt.Errorf("failed to generate pre-signed url: %w", err) + } + + _, err = FormsQ(r).Insert(&data.Form{ + ID: id, + Status: data.CreatedStatus, + Nullifier: nullifier, }) + if err != nil { + return "", "", fmt.Errorf("failed to insert created form: %w", err) + } + + return signedURL, id, nil } diff --git a/internal/service/requests/submit_form.go b/internal/service/requests/submit_form.go index c31a715..3b9f8f3 100644 --- a/internal/service/requests/submit_form.go +++ b/internal/service/requests/submit_form.go @@ -6,7 +6,7 @@ import ( "regexp" validation "github.com/go-ozzo/ozzo-validation/v4" - "github.com/go-ozzo/ozzo-validation/v4/is" + "github.com/rarimo/geo-forms-svc/internal/storage" "github.com/rarimo/geo-forms-svc/resources" ) @@ -33,7 +33,7 @@ func NewSubmitForm(r *http.Request) (req resources.SubmitFormRequest, err error) "data/attributes/postal": validation.Validate(req.Data.Attributes.Postal, validation.Required), "data/attributes/phone": validation.Validate(req.Data.Attributes.Phone, validation.Required), "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(emailRegexp)), - "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.URL), + "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, validation.Match(storage.DOSpacesURLRegexp)), } return req, errs.Filter() diff --git a/internal/service/workers/formsender/main.go b/internal/service/workers/formsender/main.go index 938ebbc..41280d3 100644 --- a/internal/service/workers/formsender/main.go +++ b/internal/service/workers/formsender/main.go @@ -55,7 +55,10 @@ func Run(ctx context.Context, cfg config.Config) { ids[i] = v.ID } - if err = db.FormsQ().FilterByID(ids...).Update(data.ProcessedStatus); err != nil { + err = db.FormsQ().FilterByID(ids...).Update(map[string]any{ + data.ColStatus: data.ProcessedStatus, + }) + if err != nil { return fmt.Errorf("failed to update form status: %w", err) } diff --git a/internal/storage/main.go b/internal/storage/main.go index 0f9c31e..13544df 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -37,14 +37,18 @@ func (s *Storage) GetImageBase64(object *url.URL) (*string, error) { return &imageBase64, nil } -func (s *Storage) ValidateImage(object *url.URL) error { +func (s *Storage) ValidateImage(object *url.URL, id string) error { spacesURL, err := parseDOSpacesURL(object) if err != nil { return fmt.Errorf("failed to parse url [%s]: %w", object.String(), err) } if spacesURL.Bucket != s.bucket { - return ErrBucketNotAllowed + return ErrInvalidBucket + } + + if spacesURL.Key != id { + return ErrInvalidKey } // output can't be nil @@ -67,8 +71,11 @@ func (s *Storage) ValidateImage(object *url.URL) error { return nil } -func (s *Storage) GeneratePutURL(contentType string, contentLength int64) (signedURL, key string, err error) { +func (s *Storage) GeneratePutURL(fileName, contentType string, contentLength int64) (signedURL, key string, err error) { key = uuid.New().String() + if fileName != "" { + key = fileName + } req, _ := s.client.PutObjectRequest(&s3.PutObjectInput{ Bucket: &s.bucket, Key: &key, @@ -89,7 +96,7 @@ func parseDOSpacesURL(object *url.URL) (*SpacesURL, error) { URL: object, } - components := doSpacesURLRegexp.FindStringSubmatch(object.String()) + components := DOSpacesURLRegexp.FindStringSubmatch(object.String()) if components == nil { return nil, ErrURLRegexp } @@ -103,10 +110,11 @@ func parseDOSpacesURL(object *url.URL) (*SpacesURL, error) { } func IsBadRequestError(err error) bool { - if errors.Is(err, ErrImageTooLarge) && - errors.Is(err, ErrIncorrectImageType) && - errors.Is(err, ErrURLRegexp) && - errors.Is(err, ErrBucketNotAllowed) { + if errors.Is(err, ErrImageTooLarge) || + errors.Is(err, ErrIncorrectImageType) || + errors.Is(err, ErrURLRegexp) || + errors.Is(err, ErrInvalidBucket) || + errors.Is(err, ErrInvalidKey) { return true } return false diff --git a/internal/storage/types.go b/internal/storage/types.go index 57271cc..4262e29 100644 --- a/internal/storage/types.go +++ b/internal/storage/types.go @@ -9,7 +9,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) -var doSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) +var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) const maxImageSize = 1 << 22 // 4mb @@ -17,7 +17,8 @@ var ( ErrImageTooLarge = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize) ErrIncorrectImageType = fmt.Errorf("incorrect object type, must be image/png or image/jpeg") ErrURLRegexp = fmt.Errorf("url don't match regexp") - ErrBucketNotAllowed = fmt.Errorf("bucket not allowed") + ErrInvalidBucket = fmt.Errorf("invalid bucket") + ErrInvalidKey = fmt.Errorf("invalid key") defaultPresignedURLExpiration = 5 * time.Minute )