From eb79a08d40d155bf765f4f3022eb030cb11db4a0 Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 11:26:23 +0300 Subject: [PATCH 1/8] Add endpoints with s3 integrations. Change docs. --- README.md | 2 + config.yaml | 5 + .../components/parameters/pathNullifier.yaml | 8 + docs/spec/components/schemas/FormStatus.yaml | 41 ++++ .../components/schemas/FormStatusKey.yaml | 7 + docs/spec/components/schemas/SubmitForm.yaml | 9 +- .../components/schemas/SubmitFormKey.yaml | 6 - .../securitySchemes/BearerAuth.yaml | 7 +- ...s@forms-svc@v1@form@lightweightsubmit.yaml | 44 +++++ ...integrations@forms-svc@v1@form@submit.yaml | 8 +- .../integrations@forms-svc@v1@form@{id}.yaml | 9 +- ...ns@forms-svc@v1@form@{nullifier}@last.yaml | 31 +++ go.mod | 12 +- go.sum | 25 ++- internal/assets/migrations/003_image_url.sql | 16 ++ internal/config/main.go | 4 +- internal/config/storage.go | 178 ++++++++++++++++++ internal/data/forms.go | 57 +++--- internal/data/pg/forms.go | 38 ++-- internal/service/handlers/ctx.go | 11 ++ internal/service/handlers/form_by_id.go | 47 +++++ .../service/handlers/form_by_nullifier.go | 72 +++++++ internal/service/handlers/get_form.go | 38 ---- internal/service/handlers/submit_form.go | 31 ++- .../handlers/submit_lightweight_form.go | 94 +++++++++ .../requests/{get_form.go => form_by_id.go} | 10 +- .../service/requests/form_by_nullifier.go | 20 ++ .../requests/submit_lightweight_form.go | 38 ++++ internal/service/router.go | 5 +- internal/service/workers/formsender/main.go | 18 ++ resources/model_form_attributes.go | 5 +- resources/model_form_status.go | 43 +++++ resources/model_form_status_attributes.go | 18 ++ resources/model_resource_type.go | 1 + resources/model_submit_form_attributes.go | 5 +- 35 files changed, 821 insertions(+), 142 deletions(-) create mode 100644 docs/spec/components/parameters/pathNullifier.yaml create mode 100644 docs/spec/components/schemas/FormStatus.yaml create mode 100644 docs/spec/components/schemas/FormStatusKey.yaml create mode 100644 docs/spec/paths/integrations@forms-svc@v1@form@lightweightsubmit.yaml create mode 100644 docs/spec/paths/integrations@forms-svc@v1@form@{nullifier}@last.yaml create mode 100644 internal/assets/migrations/003_image_url.sql create mode 100644 internal/config/storage.go create mode 100644 internal/service/handlers/form_by_id.go create mode 100644 internal/service/handlers/form_by_nullifier.go delete mode 100644 internal/service/handlers/get_form.go create mode 100644 internal/service/handlers/submit_lightweight_form.go rename internal/service/requests/{get_form.go => form_by_id.go} (52%) create mode 100644 internal/service/requests/form_by_nullifier.go create mode 100644 internal/service/requests/submit_lightweight_form.go create mode 100644 resources/model_form_status.go create mode 100644 resources/model_form_status_attributes.go diff --git a/README.md b/README.md index bc8eb97..67f2c39 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ cd geo-forms-svc go build main.go export KV_VIPER_FILE=./config.yaml + export SPACES_KEY=access_key + export SPACES_SECRET=secret_key ./main migrate up ./main run service ``` diff --git a/config.yaml b/config.yaml index 88b72cb..9296945 100644 --- a/config.yaml +++ b/config.yaml @@ -16,5 +16,10 @@ forms: resend_forms_count: 1 url: forms:forms@tcp(127.0.0.1:5441)/forms +storage: + endpoint: https://fra1.digitaloceanspaces.com + allowed_buckets: + - bucket + auth: addr: http://127.0.0.1:5000 diff --git a/docs/spec/components/parameters/pathNullifier.yaml b/docs/spec/components/parameters/pathNullifier.yaml new file mode 100644 index 0000000..68149ca --- /dev/null +++ b/docs/spec/components/parameters/pathNullifier.yaml @@ -0,0 +1,8 @@ +in: path +name: 'nullifier' +required: true +schema: + type: string + description: User nullifier + example: "0x123...abc" + pattern: '^0x[0-9a-fA-F]{64}$' diff --git a/docs/spec/components/schemas/FormStatus.yaml b/docs/spec/components/schemas/FormStatus.yaml new file mode 100644 index 0000000..533530d --- /dev/null +++ b/docs/spec/components/schemas/FormStatus.yaml @@ -0,0 +1,41 @@ +allOf: + - $ref: '#/components/schemas/FormStatusKey' + - type: object + required: + - attributes + properties: + attributes: + type: object + required: + - status + - created_at + - next_form_at + - until_next_form + properties: + status: + type: string + enum: [ accepted, processed ] + description: | + Accepted - the data was saved by the service for further processing + Processed - the data is processed and stored + created_at: + type: integer + format: int64 + example: 1721392530 + description: Form submission time. Unix UTC time. + processed_at: + type: integer + format: int64 + example: 1721392530 + description: Form processing time. Absent if the status is accepted. Unix UTC time. + next_form_at: + type: integer + format: int64 + example: 1721392530 + description: Time of the next possible form submission. Unix UTC time. + until_next_form: + type: integer + format: int64 + example: 120 + description: Time until the next form submission in seconds. + diff --git a/docs/spec/components/schemas/FormStatusKey.yaml b/docs/spec/components/schemas/FormStatusKey.yaml new file mode 100644 index 0000000..f88f65c --- /dev/null +++ b/docs/spec/components/schemas/FormStatusKey.yaml @@ -0,0 +1,7 @@ +type: object +required: + - type +properties: + type: + type: string + enum: [ form_status ] diff --git a/docs/spec/components/schemas/SubmitForm.yaml b/docs/spec/components/schemas/SubmitForm.yaml index 97c95e0..7e45738 100644 --- a/docs/spec/components/schemas/SubmitForm.yaml +++ b/docs/spec/components/schemas/SubmitForm.yaml @@ -31,10 +31,12 @@ allOf: type: string birthday: type: string + description: Date formated as DD/MM/YYYY citizen: type: string visited: type: string + description: Date formated as DD/MM/YYYY purpose: type: string country: @@ -49,6 +51,11 @@ allOf: type: string email: type: string + pattern: '[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,64}' image: type: string - description: base64 encoded image with max size 4 MB + description: | + For default endpoint: + base64 encoded image with max size 4 MB; + For lightweight endpoint: + link to the image in s3 storage; diff --git a/docs/spec/components/schemas/SubmitFormKey.yaml b/docs/spec/components/schemas/SubmitFormKey.yaml index 5bb31e8..d61e8cb 100644 --- a/docs/spec/components/schemas/SubmitFormKey.yaml +++ b/docs/spec/components/schemas/SubmitFormKey.yaml @@ -1,13 +1,7 @@ type: object required: - - id - type properties: - id: - type: string - example: "0x123...abc" - pattern: '^0x[0-9a-fA-F]{64}$' - description: User nullifier 32 bytes type: type: string enum: [ submit_form ] diff --git a/docs/spec/components/securitySchemes/BearerAuth.yaml b/docs/spec/components/securitySchemes/BearerAuth.yaml index a4a8015..df0ca1c 100644 --- a/docs/spec/components/securitySchemes/BearerAuth.yaml +++ b/docs/spec/components/securitySchemes/BearerAuth.yaml @@ -1,4 +1,3 @@ -BearerAuth: - type: http - scheme: bearer - bearerFormat: JWT +type: http +scheme: bearer +bearerFormat: JWT diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@lightweightsubmit.yaml b/docs/spec/paths/integrations@forms-svc@v1@form@lightweightsubmit.yaml new file mode 100644 index 0000000..035865c --- /dev/null +++ b/docs/spec/paths/integrations@forms-svc@v1@form@lightweightsubmit.yaml @@ -0,0 +1,44 @@ +post: + tags: + - User form + summary: Submit lightweight user answers + description: | + Send user answers and return their current status. + + The image is a link to s3 storage + + Only a user with a confirmed passport can send the form ('verified: true' in JWT). + There is a configurable delay before the same user + can submit another form. + operationId: lightweightSubmitForm + security: + - BearerAuth: [] + requestBody: + content: + application/vnd.api+json: + schema: + type: object + required: + - data + properties: + data: + $ref: '#/components/schemas/SubmitForm' + responses: + 200: + description: "Success" + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/FormStatus' + 400: + $ref: '#/components/responses/invalidParameter' + 401: + $ref: '#/components/responses/invalidAuth' + 429: + description: "It is necessary to wait some time before sending the next form" + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/Errors' + 500: + $ref: '#/components/responses/internalError' diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml b/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml index 2759714..07527ce 100644 --- a/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml +++ b/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml @@ -5,12 +5,14 @@ post: description: | Send user answers and return their current status. - Only a user with a confirmed passport can send the form. + The image is a base64 string. + + Only a user with a confirmed passport can send the form ('verified: true' in JWT). There is a configurable delay before the same user can submit another form. operationId: submitForm security: - - BearerAuth: [ "verified: true" ] + - BearerAuth: [] requestBody: content: application/vnd.api+json: @@ -23,7 +25,7 @@ post: $ref: '#/components/schemas/SubmitForm' responses: 200: - description: "Success" + description: "Success. All fields except status will be empty." content: application/vnd.api+json: schema: diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@{id}.yaml b/docs/spec/paths/integrations@forms-svc@v1@form@{id}.yaml index f60e05e..ab6a60f 100644 --- a/docs/spec/paths/integrations@forms-svc@v1@form@{id}.yaml +++ b/docs/spec/paths/integrations@forms-svc@v1@form@{id}.yaml @@ -1,13 +1,14 @@ get: tags: - User form - summary: Get form + summary: Get form status description: | - Get user form by UUID with status. - User must be authorized, but `is_verified` field is not required in JWT. - operationId: getForm + Get form status by UUID. + operationId: getFormStatus parameters: - $ref: '#/components/parameters/pathID' + security: + - BearerAuth: [] responses: 200: description: Success diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@{nullifier}@last.yaml b/docs/spec/paths/integrations@forms-svc@v1@form@{nullifier}@last.yaml new file mode 100644 index 0000000..80ad991 --- /dev/null +++ b/docs/spec/paths/integrations@forms-svc@v1@form@{nullifier}@last.yaml @@ -0,0 +1,31 @@ +get: + tags: + - User form + summary: Get last form status + description: | + Get last form status filled out by the user nullifier. + operationId: getLastFormStatus + parameters: + - $ref: '#/components/parameters/pathNullifier' + security: + - BearerAuth: [] + responses: + 200: + description: Success + content: + application/vnd.api+json: + schema: + type: object + required: + - data + properties: + data: + $ref: '#/components/schemas/FormStatus' + 400: + $ref: '#/components/responses/invalidParameter' + 401: + $ref: '#/components/responses/invalidAuth' + 404: + $ref: '#/components/responses/notFound' + 500: + $ref: '#/components/responses/internalError' diff --git a/go.mod b/go.mod index 2c1415d..06c2691 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,14 @@ go 1.22.2 require ( github.com/Masterminds/squirrel v1.5.4 github.com/alecthomas/kingpin v2.2.6+incompatible + github.com/aws/aws-sdk-go v1.54.20 github.com/go-chi/chi v4.1.2+incompatible github.com/go-ozzo/ozzo-validation/v4 v4.3.0 github.com/go-sql-driver/mysql v1.6.0 github.com/rarimo/geo-auth-svc v0.1.0 github.com/rubenv/sql-migrate v1.6.1 gitlab.com/distributed_lab/ape v1.7.1 + gitlab.com/distributed_lab/dig v0.0.0-20230207152643-c44f80a4294c gitlab.com/distributed_lab/figure/v3 v3.1.4 gitlab.com/distributed_lab/kit v1.11.3 gitlab.com/distributed_lab/logan v3.8.1+incompatible @@ -46,6 +48,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect github.com/iden3/go-rapidsnark/types v0.0.3 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -77,13 +80,14 @@ require ( gitlab.com/distributed_lab/figure v2.1.2+incompatible // indirect gitlab.com/distributed_lab/lorem v0.2.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 // indirect golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.20.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 609e7ab..8952763 100644 --- a/go.sum +++ b/go.sum @@ -1201,6 +1201,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/aws/aws-sdk-go v1.54.20 h1:FZ2UcXya7bUkvkpf7TaPmiL7EubK0go1nlXGLRwEsoo= +github.com/aws/aws-sdk-go v1.54.20/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= @@ -1681,7 +1683,9 @@ github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= @@ -2123,6 +2127,8 @@ github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= gitlab.com/distributed_lab/ape v1.7.1 h1:LpTmZgG7Lvx6ulopQbH2aWI3s8ey9FsKVjbic3ZQIy4= gitlab.com/distributed_lab/ape v1.7.1/go.mod h1:Qy9Y2arL0hmZIpVpctGEFhdrVsjWtyVJ5G+bZWcFT4s= +gitlab.com/distributed_lab/dig v0.0.0-20230207152643-c44f80a4294c h1:cqPwdAw7oJpNeN0F80bg5vNI5t3oJoSCPSnC6oj+Zyw= +gitlab.com/distributed_lab/dig v0.0.0-20230207152643-c44f80a4294c/go.mod h1:NT4H8lLoIqJxFa9AM88+6uUZ38BmxnFU8VOm/LJYUF4= gitlab.com/distributed_lab/figure v2.1.2+incompatible h1:xO1KCYPK9KFx6OUBOaJ62d8vYd1R3aNgidHlC/ZtVBA= gitlab.com/distributed_lab/figure v2.1.2+incompatible/go.mod h1:tk+aPBohT49MGPLy5+eVbE1HpD/CaC5drBHfVpRI8eE= gitlab.com/distributed_lab/figure/v3 v3.1.4 h1:Wa9FtWkDcgzL53JmiZl3j17L2ugJ4o1u/5C2wiXCJgw= @@ -2202,8 +2208,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2359,8 +2365,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2545,8 +2551,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2585,8 +2591,9 @@ golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2676,8 +2683,8 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/assets/migrations/003_image_url.sql b/internal/assets/migrations/003_image_url.sql new file mode 100644 index 0000000..a83a632 --- /dev/null +++ b/internal/assets/migrations/003_image_url.sql @@ -0,0 +1,16 @@ +-- +migrate Up + +ALTER TABLE forms + ADD COLUMN image_url TEXT; +ALTER TABLE forms + ALTER COLUMN image DROP NOT NULL; +ALTER TABLE forms + ADD COLUMN updated_at TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'); + + +-- +migrate Down + +ALTER TABLE forms + DROP COLUMN image_url; +ALTER TABLE forms + DROP COLUMN updated_at; diff --git a/internal/config/main.go b/internal/config/main.go index 4e8a41f..d844d09 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -14,6 +14,7 @@ type Config interface { auth.Auther Forms() *Forms + Storage() *Storage } type config struct { @@ -22,7 +23,8 @@ type config struct { comfig.Listenerer auth.Auther - forms comfig.Once + forms comfig.Once + storage comfig.Once getter kv.Getter } diff --git a/internal/config/storage.go b/internal/config/storage.go new file mode 100644 index 0000000..7528b15 --- /dev/null +++ b/internal/config/storage.go @@ -0,0 +1,178 @@ +package config + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "net/url" + "regexp" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + + _ "github.com/go-sql-driver/mysql" + "gitlab.com/distributed_lab/dig" + "gitlab.com/distributed_lab/figure/v3" + "gitlab.com/distributed_lab/kit/kv" +) + +var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) + +const maxImageSize = 1 << 22 // 4mb + +var ( + ImageTooLargeError = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize) + IncorrectImageTypeError = fmt.Errorf("incorrect object type, must be image/png or image/jpeg") + URLRegexpError = fmt.Errorf("url don't match regexp") + BucketNotAllowedError = fmt.Errorf("bucket not allowed") +) + +type Storage struct { + client *s3.S3 + allowedBuckets []string +} + +func (c *config) Storage() *Storage { + return c.storage.Do(func() interface{} { + var envCfg struct { + SpacesKey string `dig:"SPACES_KEY,clear"` + SpacesSecret string `dig:"SPACES_SECRET,clear"` + } + + err := dig.Out(&envCfg).Now() + if err != nil { + panic(fmt.Errorf("failed to dig out spaces key and secret: %w", err)) + } + + var cfg struct { + Endpoint string `fig:"endpoint,required"` + AllowedBuckets []string `fig:"allowed_buckets,required"` + } + + err = figure.Out(&cfg). + From(kv.MustGetStringMap(c.getter, "storage")). + Please() + if err != nil { + panic(fmt.Errorf("failed to figure out s3 storage config: %w", err)) + } + + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(envCfg.SpacesKey, envCfg.SpacesSecret, ""), + Endpoint: aws.String(cfg.Endpoint), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(false), + } + + newSession, err := session.NewSession(s3Config) + if err != nil { + panic(fmt.Errorf("failed to create session: %w", err)) + } + + s3Client := s3.New(newSession) + + return &Storage{ + client: s3Client, + allowedBuckets: cfg.AllowedBuckets, + } + }).(*Storage) +} + +func (s *Storage) GetImageBase64(object *url.URL) (*string, error) { + spacesURL, err := ParseDOSpacesURL(object) + if err != nil { + return nil, fmt.Errorf("failed to parse url [%s]: %w", object.String(), err) + } + + output, err := s.client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(spacesURL.Bucket), + Key: aws.String(spacesURL.Key), + }) + if err != nil { + return nil, fmt.Errorf("failed to get object meta: %w", err) + } + defer output.Body.Close() + + imageBytes, err := io.ReadAll(output.Body) + if err != nil { + return nil, fmt.Errorf("failed to read image: %w", err) + } + + imageBase64 := base64.StdEncoding.EncodeToString(imageBytes) + + return &imageBase64, nil +} + +func (s *Storage) ValidateImage(object *url.URL) error { + spacesURL, err := ParseDOSpacesURL(object) + if err != nil { + return fmt.Errorf("failed to parse url [%s]: %w", object.String(), err) + } + + if func() error { + for _, bucket := range s.allowedBuckets { + if spacesURL.Bucket == bucket { + return nil + } + } + return BucketNotAllowedError + }() != nil { + return fmt.Errorf("bucket=%s: %w", spacesURL.Bucket, err) + } + + // output can't be nil + output, err := s.client.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(spacesURL.Bucket), + Key: aws.String(spacesURL.Key), + }) + if err != nil { + return fmt.Errorf("failed to get image meta: %w", err) + } + + if *output.ContentType != "image/jpeg" && *output.ContentType != "image/png" { + return IncorrectImageTypeError + } + + if *output.ContentLength > maxImageSize { + return ImageTooLargeError + } + + return nil +} + +type SpacesURL struct { + URL *url.URL + Bucket string + Key string + Region string +} + +func ParseDOSpacesURL(object *url.URL) (*SpacesURL, error) { + spacesURL := &SpacesURL{ + URL: object, + } + + components := DOSpacesURLRegexp.FindStringSubmatch(object.String()) + if components == nil { + return nil, URLRegexpError + } + + // never panic because of regexp validation + spacesURL.Bucket = components[1] + spacesURL.Region = components[2] + spacesURL.Key = components[3] + + return spacesURL, nil +} + +func IsBadRequestError(err error) bool { + if errors.Is(err, ImageTooLargeError) && + errors.Is(err, IncorrectImageTypeError) && + errors.Is(err, URLRegexpError) && + errors.Is(err, BucketNotAllowedError) { + return true + } + return false +} diff --git a/internal/data/forms.go b/internal/data/forms.go index 8970544..58e6b24 100644 --- a/internal/data/forms.go +++ b/internal/data/forms.go @@ -1,6 +1,7 @@ package data import ( + "database/sql" "time" ) @@ -10,38 +11,48 @@ const ( ) type Form struct { - ID string `db:"id"` - Nullifier string `db:"nullifier"` - Status string `db:"status"` - Name string `db:"name"` - Surname string `db:"surname"` - IDNum string `db:"id_num"` - Birthday string `db:"birthday"` - Citizen string `db:"citizen"` - Visited string `db:"visited"` - Purpose string `db:"purpose"` - Country string `db:"country"` - City string `db:"city"` - Address string `db:"address"` - Postal string `db:"postal"` - Phone string `db:"phone"` - Email string `db:"email"` - Image *string `db:"image"` - CreatedAt time.Time `db:"created_at"` + ID string `db:"id"` + Nullifier string `db:"nullifier"` + Status string `db:"status"` + Name string `db:"name"` + Surname string `db:"surname"` + IDNum string `db:"id_num"` + Birthday string `db:"birthday"` + Citizen string `db:"citizen"` + Visited string `db:"visited"` + Purpose string `db:"purpose"` + Country string `db:"country"` + City string `db:"city"` + Address string `db:"address"` + Postal string `db:"postal"` + Phone string `db:"phone"` + Email string `db:"email"` + Image *string `db:"image"` + ImageURL sql.NullString `db:"image_url"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +type FormStatus struct { + ID string `db:"id"` + Nullifier string `db:"nullifier"` + Status string `db:"status"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + NextFormAt time.Time } type FormsQ interface { New() FormsQ - Insert(*Form) (string, error) + Insert(*Form) (*FormStatus, error) Update(status string) error Select() ([]*Form, error) - Get() (*Form, error) - // Last returns the most recent form after applying filters. - Last() (*Form, error) Limit(uint64) FormsQ + Get(id string) (*FormStatus, error) + Last(nullifier string) (*FormStatus, error) + FilterByID(ids ...string) FormsQ - FilterByNullifier(nullifier string) FormsQ FilterByStatus(status string) FormsQ } diff --git a/internal/data/pg/forms.go b/internal/data/pg/forms.go index cd8aea2..340cb50 100644 --- a/internal/data/pg/forms.go +++ b/internal/data/pg/forms.go @@ -10,13 +10,15 @@ import ( "gitlab.com/distributed_lab/kit/pgdb" ) -const formsTable = "forms" +const ( + formsTable = "forms" + formsStatusFields = "id, nullifier, status, created_at, updated_at" +) type formsQ struct { db *pgdb.DB selector squirrel.SelectBuilder updater squirrel.UpdateBuilder - last squirrel.SelectBuilder } func NewForms(db *pgdb.DB) data.FormsQ { @@ -24,7 +26,6 @@ func NewForms(db *pgdb.DB) data.FormsQ { db: db, selector: squirrel.Select("*").From(formsTable), updater: squirrel.Update(formsTable), - last: squirrel.Select("*").From(formsTable).OrderBy("created_at DESC"), } } @@ -32,8 +33,9 @@ func (q *formsQ) New() data.FormsQ { return NewForms(q.db) } -func (q *formsQ) Insert(form *data.Form) (string, error) { - var res string +func (q *formsQ) Insert(form *data.Form) (*data.FormStatus, error) { + var res data.FormStatus + stmt := squirrel.Insert(formsTable).SetMap(map[string]interface{}{ "nullifier": form.Nullifier, "status": form.Status, @@ -51,13 +53,14 @@ func (q *formsQ) Insert(form *data.Form) (string, error) { "phone": form.Phone, "email": form.Email, "image": form.Image, - }).Suffix("RETURNING id") + "image_url": form.ImageURL, + }).Suffix("RETURNING id, nullifier, status, created_at, updated_at") if err := q.db.Get(&res, stmt); err != nil { - return "", fmt.Errorf("insert form [%+v]: %w", form, err) + return nil, fmt.Errorf("insert form: %w", err) } - return res, nil + return &res, nil } func (q *formsQ) Update(status string) error { @@ -78,27 +81,29 @@ func (q *formsQ) Select() ([]*data.Form, error) { return res, nil } -func (q *formsQ) Get() (*data.Form, error) { - var res data.Form +func (q *formsQ) Get(id string) (*data.FormStatus, error) { + var res data.FormStatus - if err := q.db.Get(&res, q.selector); err != nil { + stmt := squirrel.Select(formsStatusFields).From(formsTable).Where(squirrel.Eq{"id": id}) + if err := q.db.Get(&res, stmt); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } - return nil, fmt.Errorf("get form: %w", err) + return nil, fmt.Errorf("get form status by id=%s: %w", id, err) } return &res, nil } -func (q *formsQ) Last() (*data.Form, error) { - var res data.Form +func (q *formsQ) Last(nullifier string) (*data.FormStatus, error) { + var res data.FormStatus - if err := q.db.Get(&res, q.last); err != nil { + stmt := squirrel.Select(formsStatusFields).From(formsTable).Where(squirrel.Eq{"nullifier": nullifier}).OrderBy("created_at DESC") + if err := q.db.Get(&res, stmt); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } - return nil, fmt.Errorf("get last form: %w", err) + return nil, fmt.Errorf("get last form status by nullifier=%s: %w", nullifier, err) } return &res, nil @@ -124,6 +129,5 @@ func (q *formsQ) FilterByStatus(status string) data.FormsQ { func (q *formsQ) applyCondition(cond squirrel.Sqlizer) data.FormsQ { q.selector = q.selector.Where(cond) q.updater = q.updater.Where(cond) - q.last = q.last.Where(cond) return q } diff --git a/internal/service/handlers/ctx.go b/internal/service/handlers/ctx.go index abc8c82..59aa5a8 100644 --- a/internal/service/handlers/ctx.go +++ b/internal/service/handlers/ctx.go @@ -17,6 +17,7 @@ const ( userClaimsCtxKey formsQCtxKey formsCtxKey + storageCtxKey ) func CtxLog(entry *logan.Entry) func(context.Context) context.Context { @@ -58,3 +59,13 @@ func CtxForms(cfg *config.Forms) func(context.Context) context.Context { func Forms(r *http.Request) *config.Forms { return r.Context().Value(formsCtxKey).(*config.Forms) } + +func CtxStorage(cfg *config.Storage) func(context.Context) context.Context { + return func(ctx context.Context) context.Context { + return context.WithValue(ctx, storageCtxKey, cfg) + } +} + +func Storage(r *http.Request) *config.Storage { + return r.Context().Value(storageCtxKey).(*config.Storage) +} diff --git a/internal/service/handlers/form_by_id.go b/internal/service/handlers/form_by_id.go new file mode 100644 index 0000000..00ec4ba --- /dev/null +++ b/internal/service/handlers/form_by_id.go @@ -0,0 +1,47 @@ +package handlers + +import ( + "net/http" + + "github.com/rarimo/geo-auth-svc/pkg/auth" + "github.com/rarimo/geo-forms-svc/internal/service/requests" + "gitlab.com/distributed_lab/ape" + "gitlab.com/distributed_lab/ape/problems" +) + +func FormByID(w http.ResponseWriter, r *http.Request) { + id, err := requests.NewFormByID(r) + if err != nil { + ape.RenderErr(w, problems.BadRequest(err)...) + return + } + + formStatus, err := FormsQ(r).Get(id) + if err != nil { + Log(r).WithError(err).Error("Failed to get form") + ape.RenderErr(w, problems.InternalError()) + return + } + + if formStatus == nil { + Log(r).Debugf("Form with id=%s not found", id) + ape.RenderErr(w, problems.NotFound()) + return + } + + if !auth.Authenticates(UserClaims(r), auth.UserGrant(formStatus.Nullifier)) { + ape.RenderErr(w, problems.Unauthorized()) + return + } + + // formStatusByNullifier will never be nil because of the previous logic + lastFormStatus, err := FormsQ(r).Last(formStatus.Nullifier) + if err != nil { + Log(r).WithError(err).Error("Failed to get last form") + ape.RenderErr(w, problems.InternalError()) + return + } + formStatus.NextFormAt = lastFormStatus.CreatedAt.Add(Forms(r).Cooldown) + + ape.Render(w, newFormStatusResponse(formStatus)) +} diff --git a/internal/service/handlers/form_by_nullifier.go b/internal/service/handlers/form_by_nullifier.go new file mode 100644 index 0000000..7b014e1 --- /dev/null +++ b/internal/service/handlers/form_by_nullifier.go @@ -0,0 +1,72 @@ +package handlers + +import ( + "net/http" + "time" + + "github.com/rarimo/geo-auth-svc/pkg/auth" + "github.com/rarimo/geo-forms-svc/internal/data" + "github.com/rarimo/geo-forms-svc/internal/service/requests" + "github.com/rarimo/geo-forms-svc/resources" + "gitlab.com/distributed_lab/ape" + "gitlab.com/distributed_lab/ape/problems" +) + +func FormByNullifier(w http.ResponseWriter, r *http.Request) { + nullifier, err := requests.NewFormByNullifier(r) + if err != nil { + ape.RenderErr(w, problems.BadRequest(err)...) + return + } + + if !auth.Authenticates(UserClaims(r), auth.UserGrant(nullifier)) { + ape.RenderErr(w, problems.Unauthorized()) + return + } + + formStatus, err := FormsQ(r).Last(nullifier) + if err != nil { + Log(r).WithError(err).Error("Failed to get form by nullifier") + ape.RenderErr(w, problems.InternalError()) + return + } + + if formStatus == nil { + Log(r).Debugf("User %s doesn't have forms", nullifier) + ape.RenderErr(w, problems.NotFound()) + return + } + + formStatus.NextFormAt = formStatus.CreatedAt.Add(Forms(r).Cooldown) + + ape.Render(w, newFormStatusResponse(formStatus)) +} + +func newFormStatusResponse(formStatus *data.FormStatus) resources.FormStatusResponse { + untilNextForm := time.Now().UTC().Unix() - formStatus.NextFormAt.Unix() + if untilNextForm < 0 { + untilNextForm = 0 + } + + var processedAt *int64 + if formStatus.Status == data.ProcessedStatus { + updatedAt := formStatus.UpdatedAt.Unix() + processedAt = &updatedAt + } + + return resources.FormStatusResponse{ + Data: resources.FormStatus{ + Key: resources.Key{ + ID: formStatus.ID, + Type: resources.FORM_STATUS, + }, + Attributes: resources.FormStatusAttributes{ + Status: formStatus.Status, + CreatedAt: formStatus.CreatedAt.Unix(), + NextFormAt: formStatus.NextFormAt.Unix(), + UntilNextForm: untilNextForm, + ProcessedAt: processedAt, + }, + }, + } +} diff --git a/internal/service/handlers/get_form.go b/internal/service/handlers/get_form.go deleted file mode 100644 index 27a322c..0000000 --- a/internal/service/handlers/get_form.go +++ /dev/null @@ -1,38 +0,0 @@ -package handlers - -import ( - "net/http" - - "github.com/rarimo/geo-auth-svc/pkg/auth" - "github.com/rarimo/geo-forms-svc/internal/service/requests" - "gitlab.com/distributed_lab/ape" - "gitlab.com/distributed_lab/ape/problems" -) - -func GetForm(w http.ResponseWriter, r *http.Request) { - req, err := requests.NewGetForm(r) - if err != nil { - ape.RenderErr(w, problems.BadRequest(err)...) - return - } - - form, err := FormsQ(r).FilterByID(req.ID).Get() - if err != nil { - Log(r).WithError(err).Error("failed to get form") - ape.RenderErr(w, problems.InternalError()) - return - } - - if form == nil { - Log(r).Debugf("Form with id=%s not found", req.ID) - ape.RenderErr(w, problems.NotFound()) - return - } - - if !auth.Authenticates(UserClaims(r), auth.UserGrant(form.Nullifier)) { - ape.RenderErr(w, problems.Unauthorized()) - return - } - - ape.Render(w, newFormResponse(form.ID)) -} diff --git a/internal/service/handlers/submit_form.go b/internal/service/handlers/submit_form.go index ca4203d..656e678 100644 --- a/internal/service/handlers/submit_form.go +++ b/internal/service/handlers/submit_form.go @@ -5,7 +5,6 @@ import ( "strings" "time" - "github.com/rarimo/geo-auth-svc/pkg/auth" "github.com/rarimo/geo-forms-svc/internal/data" "github.com/rarimo/geo-forms-svc/internal/service/requests" "github.com/rarimo/geo-forms-svc/resources" @@ -20,35 +19,26 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { return } - if len(UserClaims(r)) == 0 { - ape.RenderErr(w, problems.Unauthorized()) - return - } - nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) - if !auth.Authenticates(UserClaims(r), auth.VerifiedGrant(nullifier)) { - ape.RenderErr(w, problems.Unauthorized()) - return - } - form, err := FormsQ(r).FilterByNullifier(nullifier).Last() + formStatus, err := FormsQ(r).Last(nullifier) if err != nil { Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier) ape.RenderErr(w, problems.InternalError()) return } - if form != nil { - next := form.CreatedAt.Add(Forms(r).Cooldown) + if formStatus != nil { + next := formStatus.CreatedAt.Add(Forms(r).Cooldown) if next.After(time.Now().UTC()) { - Log(r).Debugf("Form submitted time: %s; next available time: %s", form.CreatedAt.String(), next) + Log(r).Debugf("Form submitted time: %s; next available time: %s", formStatus.CreatedAt, next) ape.RenderErr(w, problems.TooManyRequests()) return } } userData := req.Data.Attributes - form = &data.Form{ + form := &data.Form{ Nullifier: nullifier, Status: data.ProcessedStatus, Name: userData.Name, @@ -72,23 +62,26 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { form.Status = data.AcceptedStatus } - formID, err := FormsQ(r).Insert(form) + formStatus, err = FormsQ(r).Insert(form) if err != nil { - Log(r).WithError(err).Error("failed to insert form") + Log(r).WithError(err).Error("Failed to insert form") ape.RenderErr(w, problems.InternalError()) return } - ape.Render(w, newFormResponse(formID)) + ape.Render(w, newFormResponse(formStatus.ID, formStatus.Status)) } -func newFormResponse(formID string) resources.FormResponse { +func newFormResponse(formID, formStatus string) resources.FormResponse { return resources.FormResponse{ Data: resources.Form{ Key: resources.Key{ ID: formID, Type: resources.FORM, }, + Attributes: resources.FormAttributes{ + Status: formStatus, + }, }, } } diff --git a/internal/service/handlers/submit_lightweight_form.go b/internal/service/handlers/submit_lightweight_form.go new file mode 100644 index 0000000..de2e9b2 --- /dev/null +++ b/internal/service/handlers/submit_lightweight_form.go @@ -0,0 +1,94 @@ +package handlers + +import ( + "database/sql" + "net/http" + "net/url" + "strings" + "time" + + validation "github.com/go-ozzo/ozzo-validation/v4" + "github.com/rarimo/geo-forms-svc/internal/config" + "github.com/rarimo/geo-forms-svc/internal/data" + "github.com/rarimo/geo-forms-svc/internal/service/requests" + "gitlab.com/distributed_lab/ape" + "gitlab.com/distributed_lab/ape/problems" +) + +func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { + req, err := requests.NewSubmitLightweightForm(r) + if err != nil { + ape.RenderErr(w, problems.BadRequest(err)...) + return + } + + nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) + + formStatus, err := FormsQ(r).Last(nullifier) + if err != nil { + Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier) + ape.RenderErr(w, problems.InternalError()) + return + } + + if formStatus != nil { + next := formStatus.CreatedAt.Add(Forms(r).Cooldown) + if next.After(time.Now().UTC()) { + Log(r).Debugf("Form submitted time: %s; next available time: %s", formStatus.CreatedAt, next) + ape.RenderErr(w, problems.TooManyRequests()) + return + } + } + + imageURL, err := url.Parse(req.Data.Attributes.Image) + if err != nil { + Log(r).WithError(err).Errorf("Failed to parse image URL %s", req.Data.Attributes.Image) + ape.RenderErr(w, problems.InternalError()) + return + } + + if err = Storage(r).ValidateImage(imageURL); err != nil { + if config.IsBadRequestError(err) { + ape.RenderErr(w, problems.BadRequest(validation.Errors{ + "image": err, + })...) + return + } + + Log(r).WithError(err).Error("Failed to validate image") + ape.RenderErr(w, problems.InternalError()) + return + } + + userData := req.Data.Attributes + form := &data.Form{ + Nullifier: nullifier, + Status: data.AcceptedStatus, + Name: userData.Name, + Surname: userData.Surname, + IDNum: userData.IdNum, + Birthday: userData.Birthday, + Citizen: userData.Citizen, + Visited: userData.Visited, + Purpose: userData.Purpose, + Country: userData.Country, + City: userData.City, + Address: userData.Address, + Postal: userData.Postal, + Phone: userData.Phone, + Email: userData.Email, + Image: nil, + ImageURL: sql.NullString{userData.Image, true}, + } + + formStatus, err = FormsQ(r).Insert(form) + if err != nil { + Log(r).WithError(err).Error("failed to insert form") + ape.RenderErr(w, problems.InternalError()) + return + } + + formStatus.NextFormAt = formStatus.CreatedAt.Add(Forms(r).Cooldown) + + ape.Render(w, newFormStatusResponse(formStatus)) +} diff --git a/internal/service/requests/get_form.go b/internal/service/requests/form_by_id.go similarity index 52% rename from internal/service/requests/get_form.go rename to internal/service/requests/form_by_id.go index 46604bb..f746e6d 100644 --- a/internal/service/requests/get_form.go +++ b/internal/service/requests/form_by_id.go @@ -8,15 +8,11 @@ import ( "github.com/go-ozzo/ozzo-validation/v4/is" ) -type GetForm struct { - ID string -} - -func NewGetForm(r *http.Request) (req GetForm, err error) { - req.ID = chi.URLParam(r, "id") +func NewFormByID(r *http.Request) (id string, err error) { + id = chi.URLParam(r, "id") err = validation.Errors{ - "id": validation.Validate(req.ID, validation.Required, is.UUID), + "id": validation.Validate(id, validation.Required, is.UUID), }.Filter() return } diff --git a/internal/service/requests/form_by_nullifier.go b/internal/service/requests/form_by_nullifier.go new file mode 100644 index 0000000..6ccfcfa --- /dev/null +++ b/internal/service/requests/form_by_nullifier.go @@ -0,0 +1,20 @@ +package requests + +import ( + "net/http" + "regexp" + + "github.com/go-chi/chi" + validation "github.com/go-ozzo/ozzo-validation/v4" +) + +var NullifierRegexp = regexp.MustCompile("^0x[0-9a-fA-F]{64}$") + +func NewFormByNullifier(r *http.Request) (nullifier string, err error) { + nullifier = chi.URLParam(r, "nullifier") + + err = validation.Errors{ + "nullifier": validation.Validate(nullifier, validation.Required, validation.Match(NullifierRegexp)), + }.Filter() + return +} diff --git a/internal/service/requests/submit_lightweight_form.go b/internal/service/requests/submit_lightweight_form.go new file mode 100644 index 0000000..cefa07d --- /dev/null +++ b/internal/service/requests/submit_lightweight_form.go @@ -0,0 +1,38 @@ +package requests + +import ( + "encoding/json" + "net/http" + "regexp" + + validation "github.com/go-ozzo/ozzo-validation/v4" + "github.com/go-ozzo/ozzo-validation/v4/is" + "github.com/rarimo/geo-forms-svc/resources" +) + +func NewSubmitLightweightForm(r *http.Request) (req resources.SubmitFormRequest, err error) { + if err = json.NewDecoder(r.Body).Decode(&req); err != nil { + err = newDecodeError("body", err) + return + } + + errs := validation.Errors{ + "data/type": validation.Validate(req.Data.Type, validation.Required, validation.In(resources.SUBMIT_FORM)), + "data/attributes/name": validation.Validate(req.Data.Attributes.Name, validation.Required), + "data/attributes/surname": validation.Validate(req.Data.Attributes.Surname, validation.Required), + "data/attributes/id_num": validation.Validate(req.Data.Attributes.IdNum, validation.Required), + "data/attributes/birthday": validation.Validate(req.Data.Attributes.Birthday, validation.Required), + "data/attributes/citizen": validation.Validate(req.Data.Attributes.Citizen, validation.Required), + "data/attributes/visited": validation.Validate(req.Data.Attributes.Visited, validation.Required), + "data/attributes/purpose": validation.Validate(req.Data.Attributes.Purpose, validation.Required), + "data/attributes/country": validation.Validate(req.Data.Attributes.Country, validation.Required), + "data/attributes/city": validation.Validate(req.Data.Attributes.City, validation.Required), + "data/attributes/address": validation.Validate(req.Data.Attributes.Address, validation.Required), + "data/attributes/postal": validation.Validate(req.Data.Attributes.Postal, validation.Required), + "data/attributes/phone": validation.Validate(req.Data.Attributes.Phone, validation.Required), + "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`))), + "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.URL), + } + + return req, errs.Filter() +} diff --git a/internal/service/router.go b/internal/service/router.go index 85e6b32..c8c6c85 100644 --- a/internal/service/router.go +++ b/internal/service/router.go @@ -20,13 +20,16 @@ func Run(ctx context.Context, cfg config.Config) { handlers.CtxLog(cfg.Log()), handlers.CtxFormsQ(pg.NewForms(cfg.DB().Clone())), handlers.CtxForms(cfg.Forms()), + handlers.CtxStorage(cfg.Storage()), ), ) r.Route("/integrations/geo-forms-svc/v1", func(r chi.Router) { r.Route("/form", func(r chi.Router) { r.Use(handlers.AuthMiddleware(cfg.Auth(), cfg.Log())) + r.Get("/{id}", handlers.FormByID) + r.Get("/{nullifier}/last", handlers.FormByNullifier) r.Post("/submit", handlers.SubmitForm) - r.Get("/{id}", handlers.GetForm) + r.Post("/lightweightsubmit", handlers.SubmitLightweightForm) }) }) diff --git a/internal/service/workers/formsender/main.go b/internal/service/workers/formsender/main.go index 3386145..f5a97fd 100644 --- a/internal/service/workers/formsender/main.go +++ b/internal/service/workers/formsender/main.go @@ -3,6 +3,7 @@ package formsender import ( "context" "fmt" + "net/url" "github.com/rarimo/geo-forms-svc/internal/config" "github.com/rarimo/geo-forms-svc/internal/data" @@ -18,6 +19,7 @@ type formsQ struct { func Run(ctx context.Context, cfg config.Config) { log := cfg.Log().WithField("who", "form-sender") db := formsQ{cfg.DB().Clone()} + storage := cfg.Storage() running.WithBackOff(ctx, log, "resender", func(context.Context) error { forms, err := db.FormsQ().FilterByStatus(data.AcceptedStatus).Limit(cfg.Forms().ResendFormsCount).Select() @@ -28,6 +30,22 @@ func Run(ctx context.Context, cfg config.Config) { return nil } + for i := range forms { + if forms[i].Image != nil { + continue + } + + imageURL, err := url.Parse(forms[i].ImageURL.String) + if err != nil { + return fmt.Errorf("failed to parse image url: %w", err) + } + + forms[i].Image, err = storage.GetImageBase64(imageURL) + if err != nil { + return fmt.Errorf("failed to get image base64: %w", err) + } + } + if err = cfg.Forms().SendForms(forms...); err != nil { return fmt.Errorf("failed to send forms: %w", err) } diff --git a/resources/model_form_attributes.go b/resources/model_form_attributes.go index b1672dc..4d52015 100644 --- a/resources/model_form_attributes.go +++ b/resources/model_form_attributes.go @@ -5,15 +5,14 @@ package resources type FormAttributes struct { - Address string `json:"address"` - // DD-MM-YYYY format + Address string `json:"address"` Birthday string `json:"birthday"` Citizen string `json:"citizen"` City string `json:"city"` Country string `json:"country"` Email string `json:"email"` IdNum string `json:"id_num"` - // base64 encoded image with max size 12 MB + // base64 encoded image with max size 4 MB Image string `json:"image"` Name string `json:"name"` Phone string `json:"phone"` diff --git a/resources/model_form_status.go b/resources/model_form_status.go new file mode 100644 index 0000000..dc288f9 --- /dev/null +++ b/resources/model_form_status.go @@ -0,0 +1,43 @@ +/* + * GENERATED. Do not modify. Your changes might be overwritten! + */ + +package resources + +import "encoding/json" + +type FormStatus struct { + Key + Attributes FormStatusAttributes `json:"attributes"` +} +type FormStatusResponse struct { + Data FormStatus `json:"data"` + Included Included `json:"included"` +} + +type FormStatusListResponse struct { + Data []FormStatus `json:"data"` + Included Included `json:"included"` + Links *Links `json:"links"` + Meta json.RawMessage `json:"meta,omitempty"` +} + +func (r *FormStatusListResponse) PutMeta(v interface{}) (err error) { + r.Meta, err = json.Marshal(v) + return err +} + +func (r *FormStatusListResponse) GetMeta(out interface{}) error { + return json.Unmarshal(r.Meta, out) +} + +// MustFormStatus - returns FormStatus from include collection. +// if entry with specified key does not exist - returns nil +// if entry with specified key exists but type or ID mismatches - panics +func (c *Included) MustFormStatus(key Key) *FormStatus { + var formStatus FormStatus + if c.tryFindEntry(key, &formStatus) { + return &formStatus + } + return nil +} diff --git a/resources/model_form_status_attributes.go b/resources/model_form_status_attributes.go new file mode 100644 index 0000000..d6cef10 --- /dev/null +++ b/resources/model_form_status_attributes.go @@ -0,0 +1,18 @@ +/* + * GENERATED. Do not modify. Your changes might be overwritten! + */ + +package resources + +type FormStatusAttributes struct { + // Form submission time. Unix UTC time. + CreatedAt int64 `json:"created_at"` + // Time of the next possible form submission. Unix UTC time. + NextFormAt int64 `json:"next_form_at"` + // Form processing time. Absent if the status is accepted. Unix UTC time. + ProcessedAt *int64 `json:"processed_at,omitempty"` + // Accepted - the data was saved by the service for further processing Processed - the data is processed and stored + Status string `json:"status"` + // Time until the next form submission in seconds. + UntilNextForm int64 `json:"until_next_form"` +} diff --git a/resources/model_resource_type.go b/resources/model_resource_type.go index 57877b7..3b3a2a7 100644 --- a/resources/model_resource_type.go +++ b/resources/model_resource_type.go @@ -9,5 +9,6 @@ type ResourceType string // List of ResourceType const ( FORM ResourceType = "form" + FORM_STATUS ResourceType = "form_status" SUBMIT_FORM ResourceType = "submit_form" ) diff --git a/resources/model_submit_form_attributes.go b/resources/model_submit_form_attributes.go index 2a91788..a193947 100644 --- a/resources/model_submit_form_attributes.go +++ b/resources/model_submit_form_attributes.go @@ -6,19 +6,20 @@ package resources type SubmitFormAttributes struct { Address string `json:"address"` - // DD-MM-YYYY format + // Date formated as DD/MM/YYYY Birthday string `json:"birthday"` Citizen string `json:"citizen"` City string `json:"city"` Country string `json:"country"` Email string `json:"email"` IdNum string `json:"id_num"` - // base64 encoded image with max size 12 MB + // For default endpoint: base64 encoded image with max size 4 MB; For lightweight endpoint: link to the image in s3 storage; Image string `json:"image"` Name string `json:"name"` Phone string `json:"phone"` Postal string `json:"postal"` Purpose string `json:"purpose"` Surname string `json:"surname"` + // Date formated as DD/MM/YYYY Visited string `json:"visited"` } From 94ea951866cd1a7fbe3be17e008cec457194d3ae Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 11:38:50 +0300 Subject: [PATCH 2/8] Change error names, remove unncecessary imports --- internal/config/storage.go | 25 +++++++++---------- .../handlers/submit_lightweight_form.go | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/config/storage.go b/internal/config/storage.go index 7528b15..9da5c46 100644 --- a/internal/config/storage.go +++ b/internal/config/storage.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - _ "github.com/go-sql-driver/mysql" "gitlab.com/distributed_lab/dig" "gitlab.com/distributed_lab/figure/v3" "gitlab.com/distributed_lab/kit/kv" @@ -24,10 +23,10 @@ var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.d const maxImageSize = 1 << 22 // 4mb var ( - ImageTooLargeError = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize) - IncorrectImageTypeError = fmt.Errorf("incorrect object type, must be image/png or image/jpeg") - URLRegexpError = fmt.Errorf("url don't match regexp") - BucketNotAllowedError = fmt.Errorf("bucket not allowed") + ErrImageTooLarge = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize) + ErrIncorrectImageType = fmt.Errorf("incorrect object type, must be image/png or image/jpeg") + ErrURLRegexp = fmt.Errorf("url don't match regexp") + ErrBucketNotAllowed = fmt.Errorf("bucket not allowed") ) type Storage struct { @@ -117,7 +116,7 @@ func (s *Storage) ValidateImage(object *url.URL) error { return nil } } - return BucketNotAllowedError + return ErrBucketNotAllowed }() != nil { return fmt.Errorf("bucket=%s: %w", spacesURL.Bucket, err) } @@ -132,11 +131,11 @@ func (s *Storage) ValidateImage(object *url.URL) error { } if *output.ContentType != "image/jpeg" && *output.ContentType != "image/png" { - return IncorrectImageTypeError + return ErrIncorrectImageType } if *output.ContentLength > maxImageSize { - return ImageTooLargeError + return ErrImageTooLarge } return nil @@ -156,7 +155,7 @@ func ParseDOSpacesURL(object *url.URL) (*SpacesURL, error) { components := DOSpacesURLRegexp.FindStringSubmatch(object.String()) if components == nil { - return nil, URLRegexpError + return nil, ErrURLRegexp } // never panic because of regexp validation @@ -168,10 +167,10 @@ func ParseDOSpacesURL(object *url.URL) (*SpacesURL, error) { } func IsBadRequestError(err error) bool { - if errors.Is(err, ImageTooLargeError) && - errors.Is(err, IncorrectImageTypeError) && - errors.Is(err, URLRegexpError) && - errors.Is(err, BucketNotAllowedError) { + if errors.Is(err, ErrImageTooLarge) && + errors.Is(err, ErrIncorrectImageType) && + errors.Is(err, ErrURLRegexp) && + errors.Is(err, ErrBucketNotAllowed) { return true } return false diff --git a/internal/service/handlers/submit_lightweight_form.go b/internal/service/handlers/submit_lightweight_form.go index de2e9b2..d03e830 100644 --- a/internal/service/handlers/submit_lightweight_form.go +++ b/internal/service/handlers/submit_lightweight_form.go @@ -78,7 +78,7 @@ func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { Phone: userData.Phone, Email: userData.Email, Image: nil, - ImageURL: sql.NullString{userData.Image, true}, + ImageURL: sql.NullString{String: userData.Image, Valid: true}, } formStatus, err = FormsQ(r).Insert(form) From 22393eb87bcbb8bd12294965ab0f655ade8e6d36 Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 13:25:15 +0300 Subject: [PATCH 3/8] Code refactoring --- .../components/parameters/pathNullifier.yaml | 8 -- docs/spec/components/schemas/Form.yaml | 21 ++++- docs/spec/components/schemas/FormKey.yaml | 3 +- docs/spec/components/schemas/FormStatus.yaml | 6 +- docs/spec/components/schemas/SubmitForm.yaml | 61 -------------- .../components/schemas/SubmitFormKey.yaml | 7 -- ...ml => integrations@forms-svc@v1@form.yaml} | 0 ...integrations@forms-svc@v1@form@submit.yaml | 4 +- ...ntegrations@forms-svc@v1@status@last.yaml} | 0 ...ntegrations@forms-svc@v1@status@{id}.yaml} | 0 internal/config/main.go | 8 +- internal/service/handlers/ctx.go | 7 +- .../{form_by_nullifier.go => last_status.go} | 16 +--- .../{form_by_id.go => status_by_id.go} | 35 +++++--- internal/service/handlers/submit_form.go | 23 ++++-- .../handlers/submit_lightweight_form.go | 14 ++-- .../service/requests/form_by_nullifier.go | 20 ----- .../{form_by_id.go => status_by_id.go} | 2 +- .../requests/submit_lightweight_form.go | 4 +- internal/service/router.go | 10 ++- internal/service/workers/formsender/main.go | 8 +- internal/storage/config.go | 74 +++++++++++++++++ .../{config/storage.go => storage/main.go} | 79 +------------------ internal/storage/types.go | 32 ++++++++ resources/model_form_attributes.go | 8 +- resources/model_form_status_attributes.go | 6 +- resources/model_resource_type.go | 2 +- 27 files changed, 215 insertions(+), 243 deletions(-) delete mode 100644 docs/spec/components/parameters/pathNullifier.yaml delete mode 100644 docs/spec/components/schemas/SubmitForm.yaml delete mode 100644 docs/spec/components/schemas/SubmitFormKey.yaml rename docs/spec/paths/{integrations@forms-svc@v1@form@lightweightsubmit.yaml => integrations@forms-svc@v1@form.yaml} (100%) rename docs/spec/paths/{integrations@forms-svc@v1@form@{nullifier}@last.yaml => integrations@forms-svc@v1@status@last.yaml} (100%) rename docs/spec/paths/{integrations@forms-svc@v1@form@{id}.yaml => integrations@forms-svc@v1@status@{id}.yaml} (100%) rename internal/service/handlers/{form_by_nullifier.go => last_status.go} (77%) rename internal/service/handlers/{form_by_id.go => status_by_id.go} (64%) delete mode 100644 internal/service/requests/form_by_nullifier.go rename internal/service/requests/{form_by_id.go => status_by_id.go} (83%) create mode 100644 internal/storage/config.go rename internal/{config/storage.go => storage/main.go} (52%) create mode 100644 internal/storage/types.go diff --git a/docs/spec/components/parameters/pathNullifier.yaml b/docs/spec/components/parameters/pathNullifier.yaml deleted file mode 100644 index 68149ca..0000000 --- a/docs/spec/components/parameters/pathNullifier.yaml +++ /dev/null @@ -1,8 +0,0 @@ -in: path -name: 'nullifier' -required: true -schema: - type: string - description: User nullifier - example: "0x123...abc" - pattern: '^0x[0-9a-fA-F]{64}$' diff --git a/docs/spec/components/schemas/Form.yaml b/docs/spec/components/schemas/Form.yaml index 08c64be..fbc6937 100644 --- a/docs/spec/components/schemas/Form.yaml +++ b/docs/spec/components/schemas/Form.yaml @@ -7,7 +7,6 @@ allOf: attributes: type: object required: - - status - name - surname - id_num @@ -29,32 +28,48 @@ allOf: description: | Accepted - the data was saved by the service for further processing Processed - the data is processed and stored + Read-only. name: type: string + example: Hilary surname: type: string + example: Cabe id_num: type: string + example: "3736297649" birthday: type: string + example: 27/6/1988 citizen: type: string + example: Georgian visited: type: string + example: 18/07/2024 purpose: type: string + example: Make documents country: type: string + example: Georgia city: type: string + example: Kutaisi address: type: string + example: Central street, 1 postal: type: string + example: "21626" phone: type: string + example: "+13165282105" email: - type: string + type: cabehilary88@gmail.com image: type: string - description: base64 encoded image with max size 4 MB + description: | + base64 encoded image with max size 4 MB or + URL for S3 storage with image up to 4 mb + example: https://geoforms.nyc3.digitaloceanspaces.com/awesome_phono.jpg diff --git a/docs/spec/components/schemas/FormKey.yaml b/docs/spec/components/schemas/FormKey.yaml index f33e01d..be841ed 100644 --- a/docs/spec/components/schemas/FormKey.yaml +++ b/docs/spec/components/schemas/FormKey.yaml @@ -1,6 +1,5 @@ type: object required: - - id - type properties: id: @@ -8,4 +7,4 @@ properties: description: UUID for check form submission status type: type: string - enum: [ form ] + enum: [ form, submit_form ] diff --git a/docs/spec/components/schemas/FormStatus.yaml b/docs/spec/components/schemas/FormStatus.yaml index 533530d..cbfd4b9 100644 --- a/docs/spec/components/schemas/FormStatus.yaml +++ b/docs/spec/components/schemas/FormStatus.yaml @@ -22,17 +22,17 @@ allOf: type: integer format: int64 example: 1721392530 - description: Form submission time. Unix UTC time. + description: Form submission time. Unix time. processed_at: type: integer format: int64 example: 1721392530 - description: Form processing time. Absent if the status is accepted. Unix UTC time. + description: Form processing time. Absent if the status is accepted. Unix time. next_form_at: type: integer format: int64 example: 1721392530 - description: Time of the next possible form submission. Unix UTC time. + description: Time of the next possible form submission. Unix time. until_next_form: type: integer format: int64 diff --git a/docs/spec/components/schemas/SubmitForm.yaml b/docs/spec/components/schemas/SubmitForm.yaml deleted file mode 100644 index 7e45738..0000000 --- a/docs/spec/components/schemas/SubmitForm.yaml +++ /dev/null @@ -1,61 +0,0 @@ -allOf: - - $ref: '#/components/schemas/SubmitFormKey' - - type: object - x-go-is-request: true - required: - - attributes - properties: - attributes: - type: object - required: - - name - - surname - - id_num - - birthday - - citizen - - visited - - purpose - - country - - city - - address - - postal - - phone - - email - - image - properties: - name: - type: string - surname: - type: string - id_num: - type: string - birthday: - type: string - description: Date formated as DD/MM/YYYY - citizen: - type: string - visited: - type: string - description: Date formated as DD/MM/YYYY - purpose: - type: string - country: - type: string - city: - type: string - address: - type: string - postal: - type: string - phone: - type: string - email: - type: string - pattern: '[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,64}' - image: - type: string - description: | - For default endpoint: - base64 encoded image with max size 4 MB; - For lightweight endpoint: - link to the image in s3 storage; diff --git a/docs/spec/components/schemas/SubmitFormKey.yaml b/docs/spec/components/schemas/SubmitFormKey.yaml deleted file mode 100644 index d61e8cb..0000000 --- a/docs/spec/components/schemas/SubmitFormKey.yaml +++ /dev/null @@ -1,7 +0,0 @@ -type: object -required: - - type -properties: - type: - type: string - enum: [ submit_form ] diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@lightweightsubmit.yaml b/docs/spec/paths/integrations@forms-svc@v1@form.yaml similarity index 100% rename from docs/spec/paths/integrations@forms-svc@v1@form@lightweightsubmit.yaml rename to docs/spec/paths/integrations@forms-svc@v1@form.yaml diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml b/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml index 07527ce..7c35710 100644 --- a/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml +++ b/docs/spec/paths/integrations@forms-svc@v1@form@submit.yaml @@ -22,10 +22,10 @@ post: - data properties: data: - $ref: '#/components/schemas/SubmitForm' + $ref: '#/components/schemas/Form' responses: 200: - description: "Success. All fields except status will be empty." + description: "Success. All fields except image will be returned." content: application/vnd.api+json: schema: diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@{nullifier}@last.yaml b/docs/spec/paths/integrations@forms-svc@v1@status@last.yaml similarity index 100% rename from docs/spec/paths/integrations@forms-svc@v1@form@{nullifier}@last.yaml rename to docs/spec/paths/integrations@forms-svc@v1@status@last.yaml diff --git a/docs/spec/paths/integrations@forms-svc@v1@form@{id}.yaml b/docs/spec/paths/integrations@forms-svc@v1@status@{id}.yaml similarity index 100% rename from docs/spec/paths/integrations@forms-svc@v1@form@{id}.yaml rename to docs/spec/paths/integrations@forms-svc@v1@status@{id}.yaml diff --git a/internal/config/main.go b/internal/config/main.go index d844d09..abae798 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -2,6 +2,7 @@ package config import ( "github.com/rarimo/geo-auth-svc/pkg/auth" + "github.com/rarimo/geo-forms-svc/internal/storage" "gitlab.com/distributed_lab/kit/comfig" "gitlab.com/distributed_lab/kit/kv" "gitlab.com/distributed_lab/kit/pgdb" @@ -12,9 +13,9 @@ type Config interface { pgdb.Databaser comfig.Listenerer auth.Auther + storage.Storager Forms() *Forms - Storage() *Storage } type config struct { @@ -22,9 +23,9 @@ type config struct { pgdb.Databaser comfig.Listenerer auth.Auther + storage.Storager - forms comfig.Once - storage comfig.Once + forms comfig.Once getter kv.Getter } @@ -36,5 +37,6 @@ func New(getter kv.Getter) Config { Listenerer: comfig.NewListenerer(getter), Logger: comfig.NewLogger(getter, comfig.LoggerOpts{}), Auther: auth.NewAuther(getter), + Storager: storage.NewStorager(getter), } } diff --git a/internal/service/handlers/ctx.go b/internal/service/handlers/ctx.go index 59aa5a8..4286045 100644 --- a/internal/service/handlers/ctx.go +++ b/internal/service/handlers/ctx.go @@ -7,6 +7,7 @@ import ( "github.com/rarimo/geo-auth-svc/resources" "github.com/rarimo/geo-forms-svc/internal/config" "github.com/rarimo/geo-forms-svc/internal/data" + "github.com/rarimo/geo-forms-svc/internal/storage" "gitlab.com/distributed_lab/logan/v3" ) @@ -60,12 +61,12 @@ func Forms(r *http.Request) *config.Forms { return r.Context().Value(formsCtxKey).(*config.Forms) } -func CtxStorage(cfg *config.Storage) func(context.Context) context.Context { +func CtxStorage(cfg *storage.Storage) func(context.Context) context.Context { return func(ctx context.Context) context.Context { return context.WithValue(ctx, storageCtxKey, cfg) } } -func Storage(r *http.Request) *config.Storage { - return r.Context().Value(storageCtxKey).(*config.Storage) +func Storage(r *http.Request) *storage.Storage { + return r.Context().Value(storageCtxKey).(*storage.Storage) } diff --git a/internal/service/handlers/form_by_nullifier.go b/internal/service/handlers/last_status.go similarity index 77% rename from internal/service/handlers/form_by_nullifier.go rename to internal/service/handlers/last_status.go index 7b014e1..faa2105 100644 --- a/internal/service/handlers/form_by_nullifier.go +++ b/internal/service/handlers/last_status.go @@ -2,27 +2,17 @@ package handlers import ( "net/http" + "strings" "time" - "github.com/rarimo/geo-auth-svc/pkg/auth" "github.com/rarimo/geo-forms-svc/internal/data" - "github.com/rarimo/geo-forms-svc/internal/service/requests" "github.com/rarimo/geo-forms-svc/resources" "gitlab.com/distributed_lab/ape" "gitlab.com/distributed_lab/ape/problems" ) -func FormByNullifier(w http.ResponseWriter, r *http.Request) { - nullifier, err := requests.NewFormByNullifier(r) - if err != nil { - ape.RenderErr(w, problems.BadRequest(err)...) - return - } - - if !auth.Authenticates(UserClaims(r), auth.UserGrant(nullifier)) { - ape.RenderErr(w, problems.Unauthorized()) - return - } +func LastStatus(w http.ResponseWriter, r *http.Request) { + nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) formStatus, err := FormsQ(r).Last(nullifier) if err != nil { diff --git a/internal/service/handlers/form_by_id.go b/internal/service/handlers/status_by_id.go similarity index 64% rename from internal/service/handlers/form_by_id.go rename to internal/service/handlers/status_by_id.go index 00ec4ba..778d490 100644 --- a/internal/service/handlers/form_by_id.go +++ b/internal/service/handlers/status_by_id.go @@ -2,6 +2,7 @@ package handlers import ( "net/http" + "strings" "github.com/rarimo/geo-auth-svc/pkg/auth" "github.com/rarimo/geo-forms-svc/internal/service/requests" @@ -9,13 +10,33 @@ import ( "gitlab.com/distributed_lab/ape/problems" ) -func FormByID(w http.ResponseWriter, r *http.Request) { - id, err := requests.NewFormByID(r) +func StatusByID(w http.ResponseWriter, r *http.Request) { + id, err := requests.NewStatusByID(r) if err != nil { ape.RenderErr(w, problems.BadRequest(err)...) return } + nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) + + lastStatus, err := FormsQ(r).Last(nullifier) + if err != nil { + Log(r).WithError(err).Error("Failed to get last form") + ape.RenderErr(w, problems.InternalError()) + return + } + + if lastStatus == nil { + Log(r).Debugf("Form for user=%s not found", nullifier) + ape.RenderErr(w, problems.NotFound()) + return + } + + if lastStatus.ID == id { + ape.Render(w, newFormStatusResponse(lastStatus)) + return + } + formStatus, err := FormsQ(r).Get(id) if err != nil { Log(r).WithError(err).Error("Failed to get form") @@ -33,15 +54,7 @@ func FormByID(w http.ResponseWriter, r *http.Request) { ape.RenderErr(w, problems.Unauthorized()) return } - - // formStatusByNullifier will never be nil because of the previous logic - lastFormStatus, err := FormsQ(r).Last(formStatus.Nullifier) - if err != nil { - Log(r).WithError(err).Error("Failed to get last form") - ape.RenderErr(w, problems.InternalError()) - return - } - formStatus.NextFormAt = lastFormStatus.CreatedAt.Add(Forms(r).Cooldown) + formStatus.NextFormAt = lastStatus.CreatedAt.Add(Forms(r).Cooldown) ape.Render(w, newFormStatusResponse(formStatus)) } diff --git a/internal/service/handlers/submit_form.go b/internal/service/handlers/submit_form.go index 656e678..00ff5aa 100644 --- a/internal/service/handlers/submit_form.go +++ b/internal/service/handlers/submit_form.go @@ -62,25 +62,38 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { form.Status = data.AcceptedStatus } - formStatus, err = FormsQ(r).Insert(form) + _, err = FormsQ(r).Insert(form) if err != nil { Log(r).WithError(err).Error("Failed to insert form") ape.RenderErr(w, problems.InternalError()) return } - ape.Render(w, newFormResponse(formStatus.ID, formStatus.Status)) + ape.Render(w, newFormResponse(form)) } -func newFormResponse(formID, formStatus string) resources.FormResponse { +func newFormResponse(form *data.Form) resources.FormResponse { return resources.FormResponse{ Data: resources.Form{ Key: resources.Key{ - ID: formID, + ID: form.ID, Type: resources.FORM, }, Attributes: resources.FormAttributes{ - Status: formStatus, + Status: &form.Status, + Address: form.Address, + Birthday: form.Birthday, + Citizen: form.Citizen, + City: form.City, + Country: form.Country, + Email: form.Email, + IdNum: form.IDNum, + Name: form.Name, + Phone: form.Phone, + Postal: form.Postal, + Purpose: form.Purpose, + Surname: form.Surname, + Visited: form.Visited, }, }, } diff --git a/internal/service/handlers/submit_lightweight_form.go b/internal/service/handlers/submit_lightweight_form.go index d03e830..1f42d47 100644 --- a/internal/service/handlers/submit_lightweight_form.go +++ b/internal/service/handlers/submit_lightweight_form.go @@ -8,9 +8,9 @@ import ( "time" validation "github.com/go-ozzo/ozzo-validation/v4" - "github.com/rarimo/geo-forms-svc/internal/config" "github.com/rarimo/geo-forms-svc/internal/data" "github.com/rarimo/geo-forms-svc/internal/service/requests" + "github.com/rarimo/geo-forms-svc/internal/storage" "gitlab.com/distributed_lab/ape" "gitlab.com/distributed_lab/ape/problems" ) @@ -24,17 +24,17 @@ func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) - formStatus, err := FormsQ(r).Last(nullifier) + lastForm, err := FormsQ(r).Last(nullifier) if err != nil { Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier) ape.RenderErr(w, problems.InternalError()) return } - if formStatus != nil { - next := formStatus.CreatedAt.Add(Forms(r).Cooldown) + if lastForm != nil { + next := lastForm.CreatedAt.Add(Forms(r).Cooldown) if next.After(time.Now().UTC()) { - Log(r).Debugf("Form submitted time: %s; next available time: %s", formStatus.CreatedAt, next) + Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next) ape.RenderErr(w, problems.TooManyRequests()) return } @@ -48,7 +48,7 @@ func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { } if err = Storage(r).ValidateImage(imageURL); err != nil { - if config.IsBadRequestError(err) { + if storage.IsBadRequestError(err) { ape.RenderErr(w, problems.BadRequest(validation.Errors{ "image": err, })...) @@ -81,7 +81,7 @@ func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { ImageURL: sql.NullString{String: userData.Image, Valid: true}, } - formStatus, err = FormsQ(r).Insert(form) + formStatus, err := FormsQ(r).Insert(form) if err != nil { Log(r).WithError(err).Error("failed to insert form") ape.RenderErr(w, problems.InternalError()) diff --git a/internal/service/requests/form_by_nullifier.go b/internal/service/requests/form_by_nullifier.go deleted file mode 100644 index 6ccfcfa..0000000 --- a/internal/service/requests/form_by_nullifier.go +++ /dev/null @@ -1,20 +0,0 @@ -package requests - -import ( - "net/http" - "regexp" - - "github.com/go-chi/chi" - validation "github.com/go-ozzo/ozzo-validation/v4" -) - -var NullifierRegexp = regexp.MustCompile("^0x[0-9a-fA-F]{64}$") - -func NewFormByNullifier(r *http.Request) (nullifier string, err error) { - nullifier = chi.URLParam(r, "nullifier") - - err = validation.Errors{ - "nullifier": validation.Validate(nullifier, validation.Required, validation.Match(NullifierRegexp)), - }.Filter() - return -} diff --git a/internal/service/requests/form_by_id.go b/internal/service/requests/status_by_id.go similarity index 83% rename from internal/service/requests/form_by_id.go rename to internal/service/requests/status_by_id.go index f746e6d..913a440 100644 --- a/internal/service/requests/form_by_id.go +++ b/internal/service/requests/status_by_id.go @@ -8,7 +8,7 @@ import ( "github.com/go-ozzo/ozzo-validation/v4/is" ) -func NewFormByID(r *http.Request) (id string, err error) { +func NewStatusByID(r *http.Request) (id string, err error) { id = chi.URLParam(r, "id") err = validation.Errors{ diff --git a/internal/service/requests/submit_lightweight_form.go b/internal/service/requests/submit_lightweight_form.go index cefa07d..51d32ec 100644 --- a/internal/service/requests/submit_lightweight_form.go +++ b/internal/service/requests/submit_lightweight_form.go @@ -10,6 +10,8 @@ import ( "github.com/rarimo/geo-forms-svc/resources" ) +var EmailRegexp = regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`) + func NewSubmitLightweightForm(r *http.Request) (req resources.SubmitFormRequest, err error) { if err = json.NewDecoder(r.Body).Decode(&req); err != nil { err = newDecodeError("body", err) @@ -30,7 +32,7 @@ func NewSubmitLightweightForm(r *http.Request) (req resources.SubmitFormRequest, "data/attributes/address": validation.Validate(req.Data.Attributes.Address, validation.Required), "data/attributes/postal": validation.Validate(req.Data.Attributes.Postal, validation.Required), "data/attributes/phone": validation.Validate(req.Data.Attributes.Phone, validation.Required), - "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`))), + "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(EmailRegexp)), "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.URL), } diff --git a/internal/service/router.go b/internal/service/router.go index c8c6c85..4c2ad07 100644 --- a/internal/service/router.go +++ b/internal/service/router.go @@ -24,12 +24,14 @@ func Run(ctx context.Context, cfg config.Config) { ), ) r.Route("/integrations/geo-forms-svc/v1", func(r chi.Router) { + r.Use(handlers.AuthMiddleware(cfg.Auth(), cfg.Log())) + r.Route("/status", func(r chi.Router) { + r.Get("/{id}", handlers.StatusByID) + r.Get("/last", handlers.LastStatus) + }) r.Route("/form", func(r chi.Router) { - r.Use(handlers.AuthMiddleware(cfg.Auth(), cfg.Log())) - r.Get("/{id}", handlers.FormByID) - r.Get("/{nullifier}/last", handlers.FormByNullifier) r.Post("/submit", handlers.SubmitForm) - r.Post("/lightweightsubmit", handlers.SubmitLightweightForm) + r.Post("/", handlers.SubmitLightweightForm) }) }) diff --git a/internal/service/workers/formsender/main.go b/internal/service/workers/formsender/main.go index f5a97fd..7066cd3 100644 --- a/internal/service/workers/formsender/main.go +++ b/internal/service/workers/formsender/main.go @@ -30,17 +30,17 @@ func Run(ctx context.Context, cfg config.Config) { return nil } - for i := range forms { - if forms[i].Image != nil { + for _, form := range forms { + if form.Image != nil { continue } - imageURL, err := url.Parse(forms[i].ImageURL.String) + imageURL, err := url.Parse(form.ImageURL.String) if err != nil { return fmt.Errorf("failed to parse image url: %w", err) } - forms[i].Image, err = storage.GetImageBase64(imageURL) + form.Image, err = storage.GetImageBase64(imageURL) if err != nil { return fmt.Errorf("failed to get image base64: %w", err) } diff --git a/internal/storage/config.go b/internal/storage/config.go new file mode 100644 index 0000000..53d8339 --- /dev/null +++ b/internal/storage/config.go @@ -0,0 +1,74 @@ +package storage + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "gitlab.com/distributed_lab/dig" + "gitlab.com/distributed_lab/figure" + "gitlab.com/distributed_lab/kit/comfig" + "gitlab.com/distributed_lab/kit/kv" +) + +type Storager interface { + Storage() *Storage +} + +func NewStorager(getter kv.Getter) Storager { + return &storager{ + getter: getter, + } +} + +type storager struct { + once comfig.Once + getter kv.Getter +} + +func (c *storager) Storage() *Storage { + return c.once.Do(func() interface{} { + var envCfg struct { + SpacesKey string `dig:"SPACES_KEY,clear"` + SpacesSecret string `dig:"SPACES_SECRET,clear"` + } + + err := dig.Out(&envCfg).Now() + if err != nil { + panic(fmt.Errorf("failed to dig out spaces key and secret: %w", err)) + } + + var cfg struct { + Endpoint string `fig:"endpoint,required"` + AllowedBuckets []string `fig:"allowed_buckets,required"` + } + + err = figure.Out(&cfg). + From(kv.MustGetStringMap(c.getter, "storage")). + Please() + if err != nil { + panic(fmt.Errorf("failed to figure out s3 storage config: %w", err)) + } + + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(envCfg.SpacesKey, envCfg.SpacesSecret, ""), + Endpoint: aws.String(cfg.Endpoint), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(false), + } + + newSession, err := session.NewSession(s3Config) + if err != nil { + panic(fmt.Errorf("failed to create session: %w", err)) + } + + s3Client := s3.New(newSession) + + return &Storage{ + client: s3Client, + allowedBuckets: cfg.AllowedBuckets, + } + }).(*Storage) +} diff --git a/internal/config/storage.go b/internal/storage/main.go similarity index 52% rename from internal/config/storage.go rename to internal/storage/main.go index 9da5c46..c9d8ad0 100644 --- a/internal/config/storage.go +++ b/internal/storage/main.go @@ -1,84 +1,16 @@ -package config +package storage import ( "encoding/base64" - "errors" "fmt" "io" "net/url" - "regexp" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - - "gitlab.com/distributed_lab/dig" - "gitlab.com/distributed_lab/figure/v3" - "gitlab.com/distributed_lab/kit/kv" + "github.com/pkg/errors" ) -var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) - -const maxImageSize = 1 << 22 // 4mb - -var ( - ErrImageTooLarge = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize) - ErrIncorrectImageType = fmt.Errorf("incorrect object type, must be image/png or image/jpeg") - ErrURLRegexp = fmt.Errorf("url don't match regexp") - ErrBucketNotAllowed = fmt.Errorf("bucket not allowed") -) - -type Storage struct { - client *s3.S3 - allowedBuckets []string -} - -func (c *config) Storage() *Storage { - return c.storage.Do(func() interface{} { - var envCfg struct { - SpacesKey string `dig:"SPACES_KEY,clear"` - SpacesSecret string `dig:"SPACES_SECRET,clear"` - } - - err := dig.Out(&envCfg).Now() - if err != nil { - panic(fmt.Errorf("failed to dig out spaces key and secret: %w", err)) - } - - var cfg struct { - Endpoint string `fig:"endpoint,required"` - AllowedBuckets []string `fig:"allowed_buckets,required"` - } - - err = figure.Out(&cfg). - From(kv.MustGetStringMap(c.getter, "storage")). - Please() - if err != nil { - panic(fmt.Errorf("failed to figure out s3 storage config: %w", err)) - } - - s3Config := &aws.Config{ - Credentials: credentials.NewStaticCredentials(envCfg.SpacesKey, envCfg.SpacesSecret, ""), - Endpoint: aws.String(cfg.Endpoint), - Region: aws.String("us-east-1"), - S3ForcePathStyle: aws.Bool(false), - } - - newSession, err := session.NewSession(s3Config) - if err != nil { - panic(fmt.Errorf("failed to create session: %w", err)) - } - - s3Client := s3.New(newSession) - - return &Storage{ - client: s3Client, - allowedBuckets: cfg.AllowedBuckets, - } - }).(*Storage) -} - func (s *Storage) GetImageBase64(object *url.URL) (*string, error) { spacesURL, err := ParseDOSpacesURL(object) if err != nil { @@ -141,13 +73,6 @@ func (s *Storage) ValidateImage(object *url.URL) error { return nil } -type SpacesURL struct { - URL *url.URL - Bucket string - Key string - Region string -} - func ParseDOSpacesURL(object *url.URL) (*SpacesURL, error) { spacesURL := &SpacesURL{ URL: object, diff --git a/internal/storage/types.go b/internal/storage/types.go new file mode 100644 index 0000000..6fc4abc --- /dev/null +++ b/internal/storage/types.go @@ -0,0 +1,32 @@ +package storage + +import ( + "fmt" + "net/url" + "regexp" + + "github.com/aws/aws-sdk-go/service/s3" +) + +var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) + +const maxImageSize = 1 << 22 // 4mb + +var ( + ErrImageTooLarge = fmt.Errorf("too large image, must be not greater than %d bytes", maxImageSize) + ErrIncorrectImageType = fmt.Errorf("incorrect object type, must be image/png or image/jpeg") + ErrURLRegexp = fmt.Errorf("url don't match regexp") + ErrBucketNotAllowed = fmt.Errorf("bucket not allowed") +) + +type Storage struct { + client *s3.S3 + allowedBuckets []string +} + +type SpacesURL struct { + URL *url.URL + Bucket string + Key string + Region string +} diff --git a/resources/model_form_attributes.go b/resources/model_form_attributes.go index 4d52015..d0470f2 100644 --- a/resources/model_form_attributes.go +++ b/resources/model_form_attributes.go @@ -18,8 +18,8 @@ type FormAttributes struct { Phone string `json:"phone"` Postal string `json:"postal"` Purpose string `json:"purpose"` - // Accepted - the data was saved by the service for further processing Processed - the data is processed and stored - Status string `json:"status"` - Surname string `json:"surname"` - Visited string `json:"visited"` + // Accepted - the data was saved by the service for further processing Processed - the data is processed and stored Read-only. + Status *string `json:"status,omitempty"` + Surname string `json:"surname"` + Visited string `json:"visited"` } diff --git a/resources/model_form_status_attributes.go b/resources/model_form_status_attributes.go index d6cef10..8b2d24e 100644 --- a/resources/model_form_status_attributes.go +++ b/resources/model_form_status_attributes.go @@ -5,11 +5,11 @@ package resources type FormStatusAttributes struct { - // Form submission time. Unix UTC time. + // Form submission time. Unix time. CreatedAt int64 `json:"created_at"` - // Time of the next possible form submission. Unix UTC time. + // Time of the next possible form submission. Unix time. NextFormAt int64 `json:"next_form_at"` - // Form processing time. Absent if the status is accepted. Unix UTC time. + // Form processing time. Absent if the status is accepted. Unix time. ProcessedAt *int64 `json:"processed_at,omitempty"` // Accepted - the data was saved by the service for further processing Processed - the data is processed and stored Status string `json:"status"` diff --git a/resources/model_resource_type.go b/resources/model_resource_type.go index 3b3a2a7..5458987 100644 --- a/resources/model_resource_type.go +++ b/resources/model_resource_type.go @@ -9,6 +9,6 @@ type ResourceType string // List of ResourceType const ( FORM ResourceType = "form" - FORM_STATUS ResourceType = "form_status" SUBMIT_FORM ResourceType = "submit_form" + FORM_STATUS ResourceType = "form_status" ) From 590b303adf244f53654457192c9a5d9e2d40c405 Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 13:45:43 +0300 Subject: [PATCH 4/8] Semantic code refactoring --- ...htweight_form.go => legacy_submit_form.go} | 80 ++++++++++--------- internal/service/handlers/submit_form.go | 76 ++++++++---------- ...htweight_form.go => legacy_submit_form.go} | 16 +++- internal/service/requests/submit_form.go | 14 +--- internal/service/router.go | 4 +- internal/storage/main.go | 6 +- 6 files changed, 98 insertions(+), 98 deletions(-) rename internal/service/handlers/{submit_lightweight_form.go => legacy_submit_form.go} (50%) rename internal/service/requests/{submit_lightweight_form.go => legacy_submit_form.go} (76%) diff --git a/internal/service/handlers/submit_lightweight_form.go b/internal/service/handlers/legacy_submit_form.go similarity index 50% rename from internal/service/handlers/submit_lightweight_form.go rename to internal/service/handlers/legacy_submit_form.go index 1f42d47..bdf7a26 100644 --- a/internal/service/handlers/submit_lightweight_form.go +++ b/internal/service/handlers/legacy_submit_form.go @@ -1,22 +1,19 @@ package handlers import ( - "database/sql" "net/http" - "net/url" "strings" "time" - validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/rarimo/geo-forms-svc/internal/data" "github.com/rarimo/geo-forms-svc/internal/service/requests" - "github.com/rarimo/geo-forms-svc/internal/storage" + "github.com/rarimo/geo-forms-svc/resources" "gitlab.com/distributed_lab/ape" "gitlab.com/distributed_lab/ape/problems" ) -func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { - req, err := requests.NewSubmitLightweightForm(r) +func LegacySubmitForm(w http.ResponseWriter, r *http.Request) { + req, err := requests.NewLegacySubmitForm(r) if err != nil { ape.RenderErr(w, problems.BadRequest(err)...) return @@ -24,46 +21,26 @@ func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) - lastForm, err := FormsQ(r).Last(nullifier) + formStatus, err := FormsQ(r).Last(nullifier) if err != nil { Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier) ape.RenderErr(w, problems.InternalError()) return } - if lastForm != nil { - next := lastForm.CreatedAt.Add(Forms(r).Cooldown) + if formStatus != nil { + next := formStatus.CreatedAt.Add(Forms(r).Cooldown) if next.After(time.Now().UTC()) { - Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next) + Log(r).Debugf("Form submitted time: %s; next available time: %s", formStatus.CreatedAt, next) ape.RenderErr(w, problems.TooManyRequests()) return } } - imageURL, err := url.Parse(req.Data.Attributes.Image) - if err != nil { - Log(r).WithError(err).Errorf("Failed to parse image URL %s", req.Data.Attributes.Image) - ape.RenderErr(w, problems.InternalError()) - return - } - - if err = Storage(r).ValidateImage(imageURL); err != nil { - if storage.IsBadRequestError(err) { - ape.RenderErr(w, problems.BadRequest(validation.Errors{ - "image": err, - })...) - return - } - - Log(r).WithError(err).Error("Failed to validate image") - ape.RenderErr(w, problems.InternalError()) - return - } - userData := req.Data.Attributes form := &data.Form{ Nullifier: nullifier, - Status: data.AcceptedStatus, + Status: data.ProcessedStatus, Name: userData.Name, Surname: userData.Surname, IDNum: userData.IdNum, @@ -77,18 +54,47 @@ func SubmitLightweightForm(w http.ResponseWriter, r *http.Request) { Postal: userData.Postal, Phone: userData.Phone, Email: userData.Email, - Image: nil, - ImageURL: sql.NullString{String: userData.Image, Valid: true}, + Image: &userData.Image, } - formStatus, err := FormsQ(r).Insert(form) + if err = Forms(r).SendForms(form); err != nil { + Log(r).WithError(err).Error("Failed to send form") + form.Status = data.AcceptedStatus + } + + _, err = FormsQ(r).Insert(form) if err != nil { - Log(r).WithError(err).Error("failed to insert form") + Log(r).WithError(err).Error("Failed to insert form") ape.RenderErr(w, problems.InternalError()) return } - formStatus.NextFormAt = formStatus.CreatedAt.Add(Forms(r).Cooldown) + ape.Render(w, newFormResponse(form)) +} - ape.Render(w, newFormStatusResponse(formStatus)) +func newFormResponse(form *data.Form) resources.FormResponse { + return resources.FormResponse{ + Data: resources.Form{ + Key: resources.Key{ + ID: form.ID, + Type: resources.FORM, + }, + Attributes: resources.FormAttributes{ + Status: &form.Status, + Address: form.Address, + Birthday: form.Birthday, + Citizen: form.Citizen, + City: form.City, + Country: form.Country, + Email: form.Email, + IdNum: form.IDNum, + Name: form.Name, + Phone: form.Phone, + Postal: form.Postal, + Purpose: form.Purpose, + Surname: form.Surname, + Visited: form.Visited, + }, + }, + } } diff --git a/internal/service/handlers/submit_form.go b/internal/service/handlers/submit_form.go index 00ff5aa..c51c0b9 100644 --- a/internal/service/handlers/submit_form.go +++ b/internal/service/handlers/submit_form.go @@ -1,13 +1,16 @@ package handlers import ( + "database/sql" "net/http" + "net/url" "strings" "time" + validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/rarimo/geo-forms-svc/internal/data" "github.com/rarimo/geo-forms-svc/internal/service/requests" - "github.com/rarimo/geo-forms-svc/resources" + "github.com/rarimo/geo-forms-svc/internal/storage" "gitlab.com/distributed_lab/ape" "gitlab.com/distributed_lab/ape/problems" ) @@ -21,26 +24,46 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { nullifier := strings.ToLower(UserClaims(r)[0].Nullifier) - formStatus, err := FormsQ(r).Last(nullifier) + lastForm, err := FormsQ(r).Last(nullifier) if err != nil { Log(r).WithError(err).Errorf("Failed to get last user form for nullifier [%s]", nullifier) ape.RenderErr(w, problems.InternalError()) return } - if formStatus != nil { - next := formStatus.CreatedAt.Add(Forms(r).Cooldown) + if lastForm != nil { + next := lastForm.CreatedAt.Add(Forms(r).Cooldown) if next.After(time.Now().UTC()) { - Log(r).Debugf("Form submitted time: %s; next available time: %s", formStatus.CreatedAt, next) + Log(r).Debugf("Form submitted time: %s; next available time: %s", lastForm.CreatedAt, next) ape.RenderErr(w, problems.TooManyRequests()) return } } + imageURL, err := url.Parse(req.Data.Attributes.Image) + if err != nil { + Log(r).WithError(err).Errorf("Failed to parse image URL %s", req.Data.Attributes.Image) + ape.RenderErr(w, problems.InternalError()) + return + } + + if err = Storage(r).ValidateImage(imageURL); err != nil { + if storage.IsBadRequestError(err) { + ape.RenderErr(w, problems.BadRequest(validation.Errors{ + "image": err, + })...) + return + } + + Log(r).WithError(err).Error("Failed to validate image") + ape.RenderErr(w, problems.InternalError()) + return + } + userData := req.Data.Attributes form := &data.Form{ Nullifier: nullifier, - Status: data.ProcessedStatus, + Status: data.AcceptedStatus, Name: userData.Name, Surname: userData.Surname, IDNum: userData.IdNum, @@ -54,47 +77,18 @@ func SubmitForm(w http.ResponseWriter, r *http.Request) { Postal: userData.Postal, Phone: userData.Phone, Email: userData.Email, - Image: &userData.Image, + Image: nil, + ImageURL: sql.NullString{String: userData.Image, Valid: true}, } - if err = Forms(r).SendForms(form); err != nil { - Log(r).WithError(err).Error("Failed to send form") - form.Status = data.AcceptedStatus - } - - _, err = FormsQ(r).Insert(form) + formStatus, err := FormsQ(r).Insert(form) if err != nil { - Log(r).WithError(err).Error("Failed to insert form") + Log(r).WithError(err).Error("failed to insert form") ape.RenderErr(w, problems.InternalError()) return } - ape.Render(w, newFormResponse(form)) -} + formStatus.NextFormAt = formStatus.CreatedAt.Add(Forms(r).Cooldown) -func newFormResponse(form *data.Form) resources.FormResponse { - return resources.FormResponse{ - Data: resources.Form{ - Key: resources.Key{ - ID: form.ID, - Type: resources.FORM, - }, - Attributes: resources.FormAttributes{ - Status: &form.Status, - Address: form.Address, - Birthday: form.Birthday, - Citizen: form.Citizen, - City: form.City, - Country: form.Country, - Email: form.Email, - IdNum: form.IDNum, - Name: form.Name, - Phone: form.Phone, - Postal: form.Postal, - Purpose: form.Purpose, - Surname: form.Surname, - Visited: form.Visited, - }, - }, - } + ape.Render(w, newFormStatusResponse(formStatus)) } diff --git a/internal/service/requests/submit_lightweight_form.go b/internal/service/requests/legacy_submit_form.go similarity index 76% rename from internal/service/requests/submit_lightweight_form.go rename to internal/service/requests/legacy_submit_form.go index 51d32ec..be27a74 100644 --- a/internal/service/requests/submit_lightweight_form.go +++ b/internal/service/requests/legacy_submit_form.go @@ -2,6 +2,7 @@ package requests import ( "encoding/json" + "fmt" "net/http" "regexp" @@ -10,9 +11,10 @@ import ( "github.com/rarimo/geo-forms-svc/resources" ) -var EmailRegexp = regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`) +// 4 b64 letters encode 3 bytes, max image size = 12 MB -> (12/3)*4 * (1 << 20) +const maxImageSize = (1 << 20) * 16 -func NewSubmitLightweightForm(r *http.Request) (req resources.SubmitFormRequest, err error) { +func NewLegacySubmitForm(r *http.Request) (req resources.SubmitFormRequest, err error) { if err = json.NewDecoder(r.Body).Decode(&req); err != nil { err = newDecodeError("body", err) return @@ -32,9 +34,15 @@ func NewSubmitLightweightForm(r *http.Request) (req resources.SubmitFormRequest, "data/attributes/address": validation.Validate(req.Data.Attributes.Address, validation.Required), "data/attributes/postal": validation.Validate(req.Data.Attributes.Postal, validation.Required), "data/attributes/phone": validation.Validate(req.Data.Attributes.Phone, validation.Required), - "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(EmailRegexp)), - "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.URL), + "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`))), + "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.Base64, validation.Length(0, maxImageSize)), } return req, errs.Filter() } + +func newDecodeError(what string, err error) error { + return validation.Errors{ + what: fmt.Errorf("decode request %s: %w", what, err), + } +} diff --git a/internal/service/requests/submit_form.go b/internal/service/requests/submit_form.go index 4b6b2c7..c31a715 100644 --- a/internal/service/requests/submit_form.go +++ b/internal/service/requests/submit_form.go @@ -2,7 +2,6 @@ package requests import ( "encoding/json" - "fmt" "net/http" "regexp" @@ -11,8 +10,7 @@ import ( "github.com/rarimo/geo-forms-svc/resources" ) -// 4 b64 letters encode 3 bytes, max image size = 12 MB -> (12/3)*4 * (1 << 20) -const maxImageSize = (1 << 20) * 16 +var emailRegexp = regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`) func NewSubmitForm(r *http.Request) (req resources.SubmitFormRequest, err error) { if err = json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -34,15 +32,9 @@ func NewSubmitForm(r *http.Request) (req resources.SubmitFormRequest, err error) "data/attributes/address": validation.Validate(req.Data.Attributes.Address, validation.Required), "data/attributes/postal": validation.Validate(req.Data.Attributes.Postal, validation.Required), "data/attributes/phone": validation.Validate(req.Data.Attributes.Phone, validation.Required), - "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(regexp.MustCompile(`[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,64}`))), - "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.Base64, validation.Length(0, maxImageSize)), + "data/attributes/email": validation.Validate(req.Data.Attributes.Email, validation.Required, validation.Match(emailRegexp)), + "data/attributes/image": validation.Validate(req.Data.Attributes.Image, validation.Required, is.URL), } return req, errs.Filter() } - -func newDecodeError(what string, err error) error { - return validation.Errors{ - what: fmt.Errorf("decode request %s: %w", what, err), - } -} diff --git a/internal/service/router.go b/internal/service/router.go index 4c2ad07..8ac52a0 100644 --- a/internal/service/router.go +++ b/internal/service/router.go @@ -30,8 +30,8 @@ func Run(ctx context.Context, cfg config.Config) { r.Get("/last", handlers.LastStatus) }) r.Route("/form", func(r chi.Router) { - r.Post("/submit", handlers.SubmitForm) - r.Post("/", handlers.SubmitLightweightForm) + r.Post("/submit", handlers.LegacySubmitForm) + r.Post("/", handlers.SubmitForm) }) }) diff --git a/internal/storage/main.go b/internal/storage/main.go index c9d8ad0..32b3668 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -12,7 +12,7 @@ import ( ) func (s *Storage) GetImageBase64(object *url.URL) (*string, error) { - spacesURL, err := ParseDOSpacesURL(object) + spacesURL, err := parseDOSpacesURL(object) if err != nil { return nil, fmt.Errorf("failed to parse url [%s]: %w", object.String(), err) } @@ -37,7 +37,7 @@ func (s *Storage) GetImageBase64(object *url.URL) (*string, error) { } func (s *Storage) ValidateImage(object *url.URL) error { - spacesURL, err := ParseDOSpacesURL(object) + spacesURL, err := parseDOSpacesURL(object) if err != nil { return fmt.Errorf("failed to parse url [%s]: %w", object.String(), err) } @@ -73,7 +73,7 @@ func (s *Storage) ValidateImage(object *url.URL) error { return nil } -func ParseDOSpacesURL(object *url.URL) (*SpacesURL, error) { +func parseDOSpacesURL(object *url.URL) (*SpacesURL, error) { spacesURL := &SpacesURL{ URL: object, } From b076af15eca109287cd5da08c60d881951b631a4 Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 13:50:27 +0300 Subject: [PATCH 5/8] Make DOSpaces regexp unexported --- internal/storage/main.go | 2 +- internal/storage/types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/storage/main.go b/internal/storage/main.go index 32b3668..f3a8c46 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -78,7 +78,7 @@ func parseDOSpacesURL(object *url.URL) (*SpacesURL, error) { URL: object, } - components := DOSpacesURLRegexp.FindStringSubmatch(object.String()) + components := doSpacesURLRegexp.FindStringSubmatch(object.String()) if components == nil { return nil, ErrURLRegexp } diff --git a/internal/storage/types.go b/internal/storage/types.go index 6fc4abc..e98d4f1 100644 --- a/internal/storage/types.go +++ b/internal/storage/types.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" ) -var DOSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) +var doSpacesURLRegexp = regexp.MustCompile(`^https:\/\/(.+?)\.(.+?)(?:\.cdn)?\.digitaloceanspaces\.com\/(.+)$`) const maxImageSize = 1 << 22 // 4mb From bf4c8ac9d66733fd8838d0b49232cd4bb2eb7d6a Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 13:56:47 +0300 Subject: [PATCH 6/8] Fix image assignment in loop --- internal/service/workers/formsender/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/service/workers/formsender/main.go b/internal/service/workers/formsender/main.go index 7066cd3..938ebbc 100644 --- a/internal/service/workers/formsender/main.go +++ b/internal/service/workers/formsender/main.go @@ -30,7 +30,7 @@ func Run(ctx context.Context, cfg config.Config) { return nil } - for _, form := range forms { + for i, form := range forms { if form.Image != nil { continue } @@ -40,7 +40,7 @@ func Run(ctx context.Context, cfg config.Config) { return fmt.Errorf("failed to parse image url: %w", err) } - form.Image, err = storage.GetImageBase64(imageURL) + forms[i].Image, err = storage.GetImageBase64(imageURL) if err != nil { return fmt.Errorf("failed to get image base64: %w", err) } From c7ca1b65faf1acbd14efd8176d3393398c984940 Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 14:02:03 +0300 Subject: [PATCH 7/8] Replace function by loop --- internal/storage/main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/storage/main.go b/internal/storage/main.go index f3a8c46..f4a1d29 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -42,15 +42,15 @@ func (s *Storage) ValidateImage(object *url.URL) error { return fmt.Errorf("failed to parse url [%s]: %w", object.String(), err) } - if func() error { - for _, bucket := range s.allowedBuckets { - if spacesURL.Bucket == bucket { - return nil - } + found := false + for _, bucket := range s.allowedBuckets { + if spacesURL.Bucket == bucket { + found = true + break } + } + if !found { return ErrBucketNotAllowed - }() != nil { - return fmt.Errorf("bucket=%s: %w", spacesURL.Bucket, err) } // output can't be nil From 948397f3bcf880a24c2b2d978ee2fe8ca523175b Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Mon, 22 Jul 2024 14:18:21 +0300 Subject: [PATCH 8/8] Add comments --- internal/storage/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/storage/config.go b/internal/storage/config.go index 53d8339..86401b5 100644 --- a/internal/storage/config.go +++ b/internal/storage/config.go @@ -28,6 +28,8 @@ type storager struct { getter kv.Getter } +// Storage works only with DigitalOceanSpaces. +// Other providers are not supported. func (c *storager) Storage() *Storage { return c.once.Do(func() interface{} { var envCfg struct {