From 0775008736473d2041419b44bf8f983cef88aa84 Mon Sep 17 00:00:00 2001 From: Zaptoss Date: Thu, 1 Aug 2024 11:04:41 +0300 Subject: [PATCH] Add spreadsheets report generator --- config.yaml | 9 +- go.mod | 27 ++- go.sum | 47 +++++ internal/cli/main.go | 9 +- internal/config/forms.go | 9 +- internal/config/main.go | 16 +- internal/data/forms.go | 2 + internal/data/pg/forms.go | 9 + .../service/handlers/legacy_submit_form.go | 7 +- internal/service/requests/upload_image.go | 4 +- internal/service/workers/formsender/main.go | 75 -------- .../service/workers/spreadsheets/config.go | 162 ++++++++++++++++++ internal/service/workers/spreadsheets/main.go | 114 ++++++++++++ internal/storage/main.go | 36 ++++ resources/model_form_status_attributes.go | 2 +- .../model_upload_image_response_attributes.go | 2 +- 16 files changed, 428 insertions(+), 102 deletions(-) delete mode 100644 internal/service/workers/formsender/main.go create mode 100644 internal/service/workers/spreadsheets/config.go create mode 100644 internal/service/workers/spreadsheets/main.go diff --git a/config.yaml b/config.yaml index 249d3cf..ef1e167 100644 --- a/config.yaml +++ b/config.yaml @@ -18,9 +18,16 @@ forms: storage: backend: "do" - endpoint: https://fra1.digitaloceanspaces.com + endpoint: https://nyc3.digitaloceanspaces.com bucket: bucket presigned_url_expiration: 3m +spreadsheets: + credentials: "./credentials.json" + token: "./token.json" + period: 1m + min_abnormal_period: 1m + max_abnormal_period: 10m + auth: addr: http://127.0.0.1:5000 diff --git a/go.mod b/go.mod index a305573..9b4dcaa 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,9 @@ require ( ) require ( + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect @@ -43,13 +46,21 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/ethereum/go-ethereum v1.13.8 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/raven-go v0.2.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/jsonapi v0.0.0-20200226002910-c8283f632fb7 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.2.4 // indirect @@ -83,15 +94,25 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect gitlab.com/distributed_lab/lorem v0.2.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/api v0.189.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect + google.golang.org/grpc v1.64.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index 122249e..555b59b 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,7 @@ cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5x cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go v0.110.9/go.mod h1:rpxevX/0Lqvlbc88b7Sc1SPNdyK1riNBTUU6JXhYNpM= cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -161,6 +162,10 @@ cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav cloud.google.com/go/assuredworkloads v1.11.2/go.mod h1:O1dfr+oZJMlE6mw0Bp0P1KZSlj5SghMBvTpZqIcUAW4= cloud.google.com/go/assuredworkloads v1.11.3/go.mod h1:vEjfTKYyRUaIeA0bsGJceFV2JKpVRgyG2op3jfa59Zs= cloud.google.com/go/assuredworkloads v1.11.4/go.mod h1:4pwwGNwy1RP0m+y12ef3Q/8PaiWrIDQ6nD2E8kvWI9U= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= @@ -309,6 +314,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -1375,6 +1382,8 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -1428,7 +1437,12 @@ github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpx github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -1482,6 +1496,7 @@ github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -1511,6 +1526,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -1571,6 +1588,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1587,6 +1605,7 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -1603,6 +1622,8 @@ github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38 github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -2165,7 +2186,16 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2219,6 +2249,8 @@ golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72 golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2376,6 +2408,8 @@ golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2411,6 +2445,8 @@ golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBch golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2562,6 +2598,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2782,6 +2820,8 @@ google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9 google.golang.org/api v0.150.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg= google.golang.org/api v0.152.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2942,6 +2982,7 @@ google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:EMfReVxb google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= @@ -2980,6 +3021,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.mod h1:67X1fPuzjcrkymZzZV1vvkFeTn2Rvc6lYF9MYFGCcwE= google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -3028,6 +3071,8 @@ google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -3049,6 +3094,8 @@ google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/cli/main.go b/internal/cli/main.go index 361a155..6ec2aeb 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -10,7 +10,7 @@ import ( "github.com/alecthomas/kingpin" "github.com/rarimo/geo-forms-svc/internal/config" "github.com/rarimo/geo-forms-svc/internal/service" - "github.com/rarimo/geo-forms-svc/internal/service/workers/formsender" + "github.com/rarimo/geo-forms-svc/internal/service/workers/spreadsheets" "gitlab.com/distributed_lab/kit/kv" "gitlab.com/distributed_lab/logan/v3" ) @@ -59,7 +59,12 @@ func Run(args []string) bool { switch cmd { case serviceCmd.FullCommand(): run(service.Run) - run(formsender.Run) + + wg.Add(1) + go func() { + spreadsheets.Run(ctx, cfg) + wg.Done() + }() case migrateUpCmd.FullCommand(): err = MigrateUp(cfg) case migrateDownCmd.FullCommand(): diff --git a/internal/config/forms.go b/internal/config/forms.go index 6992f03..1bd5d91 100644 --- a/internal/config/forms.go +++ b/internal/config/forms.go @@ -41,10 +41,10 @@ func (c *config) Forms() *Forms { panic(fmt.Errorf("failed to figure out withdrawal point price: %w", err)) } - db, err := sql.Open("mysql", cfg.URL) - if err != nil { - panic(fmt.Errorf("failed to connect to mysql: %w", err)) - } + // db, err := sql.Open("mysql", cfg.URL) + // if err != nil { + // panic(fmt.Errorf("failed to connect to mysql: %w", err)) + // } return &Forms{ Cooldown: cfg.Cooldown, @@ -52,7 +52,6 @@ func (c *config) Forms() *Forms { MinAbnormalPeriod: cfg.MinAbnormalPeriod, MaxAbnormalPeriod: cfg.MaxAbnormalPeriod, ResendFormsCount: cfg.ResendFormsCount, - db: db, } }).(*Forms) } diff --git a/internal/config/main.go b/internal/config/main.go index abae798..8133763 100644 --- a/internal/config/main.go +++ b/internal/config/main.go @@ -2,6 +2,7 @@ package config import ( "github.com/rarimo/geo-auth-svc/pkg/auth" + "github.com/rarimo/geo-forms-svc/internal/service/workers/spreadsheets" "github.com/rarimo/geo-forms-svc/internal/storage" "gitlab.com/distributed_lab/kit/comfig" "gitlab.com/distributed_lab/kit/kv" @@ -14,6 +15,7 @@ type Config interface { comfig.Listenerer auth.Auther storage.Storager + spreadsheets.Spreadsheeter Forms() *Forms } @@ -24,6 +26,7 @@ type config struct { comfig.Listenerer auth.Auther storage.Storager + spreadsheets.Spreadsheeter forms comfig.Once @@ -32,11 +35,12 @@ type config struct { func New(getter kv.Getter) Config { return &config{ - getter: getter, - Databaser: pgdb.NewDatabaser(getter), - Listenerer: comfig.NewListenerer(getter), - Logger: comfig.NewLogger(getter, comfig.LoggerOpts{}), - Auther: auth.NewAuther(getter), - Storager: storage.NewStorager(getter), + getter: getter, + Databaser: pgdb.NewDatabaser(getter), + Listenerer: comfig.NewListenerer(getter), + Logger: comfig.NewLogger(getter, comfig.LoggerOpts{}), + Auther: auth.NewAuther(getter), + Storager: storage.NewStorager(getter), + Spreadsheeter: spreadsheets.NewSpreadsheeter(getter), } } diff --git a/internal/data/forms.go b/internal/data/forms.go index 995da25..bd25913 100644 --- a/internal/data/forms.go +++ b/internal/data/forms.go @@ -75,4 +75,6 @@ type FormsQ interface { FilterByID(ids ...string) FormsQ FilterByStatus(status ...string) FormsQ + FilterByUpdatedAt(time.Time) FormsQ + FilterImages() FormsQ } diff --git a/internal/data/pg/forms.go b/internal/data/pg/forms.go index d1999cd..69ce4f1 100644 --- a/internal/data/pg/forms.go +++ b/internal/data/pg/forms.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "time" "github.com/Masterminds/squirrel" "github.com/rarimo/geo-forms-svc/internal/data" @@ -134,6 +135,14 @@ func (q *formsQ) FilterByStatus(status ...string) data.FormsQ { return q.applyCondition(squirrel.Eq{"status": status}) } +func (q *formsQ) FilterByUpdatedAt(times time.Time) data.FormsQ { + return q.applyCondition(squirrel.Gt{"updated_at": times}) +} + +func (q *formsQ) FilterImages() data.FormsQ { + return q.applyCondition(squirrel.Eq{"image": nil}) +} + func (q *formsQ) applyCondition(cond squirrel.Sqlizer) data.FormsQ { q.selector = q.selector.Where(cond) q.last = q.selector.Where(cond) diff --git a/internal/service/handlers/legacy_submit_form.go b/internal/service/handlers/legacy_submit_form.go index e75f06f..e7c1b41 100644 --- a/internal/service/handlers/legacy_submit_form.go +++ b/internal/service/handlers/legacy_submit_form.go @@ -45,7 +45,7 @@ func LegacySubmitForm(w http.ResponseWriter, r *http.Request) { userData := req.Data.Attributes form := &data.Form{ Nullifier: nullifier, - Status: data.ProcessedStatus, + Status: data.AcceptedStatus, Name: userData.Name, Surname: userData.Surname, IDNum: userData.IdNum, @@ -62,11 +62,6 @@ func LegacySubmitForm(w http.ResponseWriter, r *http.Request) { Image: &userData.Image, } - if err = Forms(r).SendForms(form); err != nil { - Log(r).WithError(err).Error("Failed to send form") - form.Status = data.AcceptedStatus - } - _, err = FormsQ(r).Insert(form) if err != nil { Log(r).WithError(err).Error("Failed to insert form") diff --git a/internal/service/requests/upload_image.go b/internal/service/requests/upload_image.go index 3a49239..5f6bc9a 100644 --- a/internal/service/requests/upload_image.go +++ b/internal/service/requests/upload_image.go @@ -8,7 +8,7 @@ import ( "github.com/rarimo/geo-forms-svc/resources" ) -const maxImageSize = 1 << 22 +const maxImageSize = int64(1 << 22) func NewUploadImage(r *http.Request) (req resources.UploadImageRequest, err error) { if err = json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -19,7 +19,7 @@ func NewUploadImage(r *http.Request) (req resources.UploadImageRequest, err erro errs := validation.Errors{ "data/type": validation.Validate(req.Data.Type, validation.Required, validation.In(resources.UPLOAD_IMAGE)), "data/attributes/content_type": validation.Validate(req.Data.Attributes.ContentType, validation.Required, validation.In("image/png", "image/jpeg")), - "data/attributes/content_length": validation.Validate(req.Data.Attributes.ContentLength, validation.Required, validation.Length(1, int(maxImageSize))), + "data/attributes/content_length": validation.Validate(req.Data.Attributes.ContentLength, validation.Required, validation.Min(int64(1)), validation.Max(maxImageSize)), } return req, errs.Filter() diff --git a/internal/service/workers/formsender/main.go b/internal/service/workers/formsender/main.go deleted file mode 100644 index 41280d3..0000000 --- a/internal/service/workers/formsender/main.go +++ /dev/null @@ -1,75 +0,0 @@ -package formsender - -import ( - "context" - "fmt" - "net/url" - - "github.com/rarimo/geo-forms-svc/internal/config" - "github.com/rarimo/geo-forms-svc/internal/data" - "github.com/rarimo/geo-forms-svc/internal/data/pg" - "gitlab.com/distributed_lab/kit/pgdb" - "gitlab.com/distributed_lab/running" -) - -type formsQ struct { - db *pgdb.DB -} - -func Run(ctx context.Context, cfg config.Config) { - log := cfg.Log().WithField("who", "form-sender") - db := formsQ{cfg.DB().Clone()} - storage := cfg.Storage() - - running.WithBackOff(ctx, log, "resender", func(context.Context) error { - forms, err := db.FormsQ().FilterByStatus(data.AcceptedStatus).Limit(cfg.Forms().ResendFormsCount).Select() - if err != nil { - return fmt.Errorf("failed to get unsended forms: %w", err) - } - if len(forms) == 0 { - return nil - } - - for i, form := range forms { - if form.Image != nil { - continue - } - - imageURL, err := url.Parse(form.ImageURL.String) - if err != nil { - return fmt.Errorf("failed to parse image url: %w", err) - } - - forms[i].Image, err = storage.GetImageBase64(imageURL) - if err != nil { - return fmt.Errorf("failed to get image base64: %w", err) - } - } - - if err = cfg.Forms().SendForms(forms...); err != nil { - return fmt.Errorf("failed to send forms: %w", err) - } - - ids := make([]string, len(forms)) - for i, v := range forms { - ids[i] = v.ID - } - - err = db.FormsQ().FilterByID(ids...).Update(map[string]any{ - data.ColStatus: data.ProcessedStatus, - }) - if err != nil { - return fmt.Errorf("failed to update form status: %w", err) - } - - return nil - }, - cfg.Forms().Period, - cfg.Forms().MinAbnormalPeriod, - cfg.Forms().MaxAbnormalPeriod, - ) -} - -func (d *formsQ) FormsQ() data.FormsQ { - return pg.NewForms(d.db) -} diff --git a/internal/service/workers/spreadsheets/config.go b/internal/service/workers/spreadsheets/config.go new file mode 100644 index 0000000..cc76c8d --- /dev/null +++ b/internal/service/workers/spreadsheets/config.go @@ -0,0 +1,162 @@ +package spreadsheets + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "time" + + "gitlab.com/distributed_lab/figure" + "gitlab.com/distributed_lab/kit/comfig" + "gitlab.com/distributed_lab/kit/kv" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" +) + +var ( + scopes = []string{ + drive.DriveFileScope, + sheets.SpreadsheetsScope, + } + + headers = []any{ + "Name", "Surname", "IDNum", "Birthday", "Citizen", + "Visited", "Purpose", "Country", "City", "Address", + "Postal", "Phone", "Email", "Time", "Image", + } + + sheetRange = "A%d:O%d" + sheetHeadersRange = fmt.Sprintf(sheetRange, 1, 1) +) + +const mimeTypeSpreadsheet = "application/vnd.google-apps.spreadsheet" + +type Spreadsheets struct { + client *http.Client + + period time.Duration + minAbnormalPeriod time.Duration + maxAbnormalPeriod time.Duration + + sheetID string + lastSubmited time.Time + + sheetsSrv *sheets.Service + driveSrv *drive.Service +} + +type Spreadsheeter interface { + Spreadsheets() *Spreadsheets +} + +func NewSpreadsheeter(getter kv.Getter) Spreadsheeter { + return &spreadsheeter{ + getter: getter, + } +} + +type spreadsheeter struct { + once comfig.Once + getter kv.Getter +} + +func (c *spreadsheeter) Spreadsheets() *Spreadsheets { + return c.once.Do(func() interface{} { + var cfg struct { + Credentials string `fig:"credentials,required"` + Token string `fig:"token,required"` + Period time.Duration `fig:"period,required"` + MinAbnormalPeriod time.Duration `fig:"min_abnormal_period,required"` + MaxAbnormalPeriod time.Duration `fig:"max_abnormal_period,required"` + } + + err := figure.Out(&cfg). + From(kv.MustGetStringMap(c.getter, "spreadsheets")). + Please() + if err != nil { + panic(fmt.Errorf("failed to figure out spreadsheets config: %w", err)) + } + + creds, err := os.ReadFile(cfg.Credentials) + if err != nil { + panic(fmt.Errorf("unable to read client secret file: %w", err)) + } + + config, err := google.ConfigFromJSON(creds, scopes...) + if err != nil { + log.Fatalf("Unable to parse client secret file to config: %v", err) + } + + tokenS, err := os.ReadFile(cfg.Token) + if err != nil { + panic(fmt.Errorf("unable to read client secret file: %w", err)) + } + + var token oauth2.Token + err = json.Unmarshal([]byte(tokenS), &token) + if err != nil { + panic(fmt.Errorf("failed to unmarshal token: %w", err)) + } + + client := config.Client(context.Background(), &token) + + sheetsSrv, err := sheets.NewService(context.Background(), option.WithHTTPClient(client)) + if err != nil { + panic(fmt.Errorf("failed to create sheets service: %w", err)) + } + + driveSrv, err := drive.NewService(context.Background(), option.WithHTTPClient(client)) + if err != nil { + panic(fmt.Errorf("failed to create drive service: %w", err)) + } + + return &Spreadsheets{ + client: client, + period: cfg.Period, + minAbnormalPeriod: cfg.MinAbnormalPeriod, + maxAbnormalPeriod: cfg.MaxAbnormalPeriod, + + sheetsSrv: sheetsSrv, + driveSrv: driveSrv, + } + }).(*Spreadsheets) +} + +func (s *Spreadsheets) CreateTable() error { + sheet, err := s.driveSrv.Files.Create(&drive.File{ + Name: time.Now().UTC().Format("01/02/2006 15:04"), + MimeType: mimeTypeSpreadsheet, + }).Do() + if err != nil { + return fmt.Errorf("failed to create spreadsheet: %w", err) + } + + _, err = s.sheetsSrv.Spreadsheets.Values.Update(sheet.Id, sheetHeadersRange, &sheets.ValueRange{ + Values: [][]any{headers}, + }).ValueInputOption("RAW").Do() + if err != nil { + return fmt.Errorf("failed to set table headers: %w", err) + } + + s.sheetID = sheet.Id + + return nil +} + +func (s *Spreadsheets) FillTable(data [][]any) error { + dataRange := len(data) + 1 + _, err := s.sheetsSrv.Spreadsheets.Values.Update(s.sheetID, fmt.Sprintf(sheetRange, 2, dataRange), &sheets.ValueRange{ + Values: data, + }).ValueInputOption("RAW").Do() + if err != nil { + return fmt.Errorf("failed to insert user data in table: %w", err) + } + + return nil +} diff --git a/internal/service/workers/spreadsheets/main.go b/internal/service/workers/spreadsheets/main.go new file mode 100644 index 0000000..cda23a6 --- /dev/null +++ b/internal/service/workers/spreadsheets/main.go @@ -0,0 +1,114 @@ +package spreadsheets + +import ( + "context" + "fmt" + "net/url" + + "github.com/rarimo/geo-forms-svc/internal/data" + "github.com/rarimo/geo-forms-svc/internal/data/pg" + "github.com/rarimo/geo-forms-svc/internal/storage" + "gitlab.com/distributed_lab/kit/comfig" + "gitlab.com/distributed_lab/kit/pgdb" + "gitlab.com/distributed_lab/running" +) + +type formsQ struct { + db *pgdb.DB +} + +type extConfig interface { + comfig.Logger + pgdb.Databaser + Spreadsheeter + storage.Storager +} + +func Run(ctx context.Context, cfg extConfig) { + log := cfg.Log().WithField("who", "spreadsheeter") + db := formsQ{cfg.DB().Clone()} + s3 := cfg.Storage() + spreadsheets := cfg.Spreadsheets() + + running.WithBackOff(ctx, log, "sheet-former", func(context.Context) error { + forms, err := db.FormsQ().FilterByStatus(data.AcceptedStatus).FilterImages().FilterByUpdatedAt(spreadsheets.lastSubmited).Select() + if err != nil { + return fmt.Errorf("failed to get unsended forms: %w", err) + } + if len(forms) == 0 { + return nil + } + + tableData := make([][]any, 0, len(forms)) + for _, form := range forms { + data := make([]any, 0, len(headers)) + + link, err := url.Parse(form.ImageURL.String) + if err != nil { + return fmt.Errorf("failed to parse image url %s: %w", form.ImageURL.String, err) + } + + signedURL, err := s3.GenerateGetURL(link) + if err != nil { + return fmt.Errorf("failed to generate pre-signed get url: %w", err) + } + + data = append(data, + form.Name, + form.Surname, + form.IDNum, + form.Birthday, + form.Citizen, + form.Visited, + form.Purpose, + form.Country, + form.City, + form.Address, + form.Postal, + form.Phone, + form.Email, + form.UpdatedAt.Format("01/02/2006 15:04"), + signedURL, + ) + + tableData = append(tableData, data) + + if form.UpdatedAt.After(spreadsheets.lastSubmited) { + spreadsheets.lastSubmited = form.UpdatedAt + } + + } + + err = spreadsheets.CreateTable() + if err != nil { + return fmt.Errorf("failed to create spreadsheet: %w", err) + } + + err = spreadsheets.FillTable(tableData) + if err != nil { + return fmt.Errorf("failed to fill spreadsheet: %w", err) + } + + ids := make([]string, len(forms)) + for i, v := range forms { + ids[i] = v.ID + } + + err = db.FormsQ().FilterByID(ids...).Update(map[string]any{ + data.ColStatus: data.ProcessedStatus, + }) + if err != nil { + return fmt.Errorf("failed to update form status: %w", err) + } + + return nil + }, + spreadsheets.period, + spreadsheets.minAbnormalPeriod, + spreadsheets.maxAbnormalPeriod, + ) +} + +func (d *formsQ) FormsQ() data.FormsQ { + return pg.NewForms(d.db) +} diff --git a/internal/storage/main.go b/internal/storage/main.go index 434150a..b4aa7a9 100644 --- a/internal/storage/main.go +++ b/internal/storage/main.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/url" + "time" "github.com/aws/amazon-ssm-agent/agent/s3util" "github.com/aws/aws-sdk-go/aws" @@ -112,6 +113,41 @@ func (s *Storage) GeneratePutURL(fileName, contentType string, contentLength int return signedURL, key, nil } +func (s *Storage) GenerateGetURL(link *url.URL) (signedURL string, err error) { + var bucket, key string + + switch s.backend { + case digitalOceanBackend: + spacesURL, err := parseDOSpacesURL(link) + if err != nil { + return "", fmt.Errorf("failed to parse url [%s]: %w", link, err) + } + key = spacesURL.Key + bucket = spacesURL.Bucket + case awsBackend: + s3URL := s3util.ParseAmazonS3URL(nil, link) + if s3URL.Region != s.region { + return "", ErrRegionMismatched + } + key = s3URL.Key + bucket = s3URL.Bucket + // should be never happened + default: + return "", errors.New("invalid backend") + } + req, _ := s.client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + + signedURL, err = req.Presign(time.Hour * 164) + if err != nil { + return "", fmt.Errorf("failed to sign request: %w", err) + } + + return signedURL, nil +} + func parseDOSpacesURL(object *url.URL) (*SpacesURL, error) { spacesURL := &SpacesURL{ URL: object, diff --git a/resources/model_form_status_attributes.go b/resources/model_form_status_attributes.go index 8b2d24e..e7b7131 100644 --- a/resources/model_form_status_attributes.go +++ b/resources/model_form_status_attributes.go @@ -11,7 +11,7 @@ type FormStatusAttributes struct { NextFormAt int64 `json:"next_form_at"` // Form processing time. Absent if the status is accepted. Unix time. ProcessedAt *int64 `json:"processed_at,omitempty"` - // Accepted - the data was saved by the service for further processing Processed - the data is processed and stored + // Created - the empty form was created and now user can't use legacy submit Accepted - the data was saved by the service for further processing Processed - the data is processed and stored Status string `json:"status"` // Time until the next form submission in seconds. UntilNextForm int64 `json:"until_next_form"` diff --git a/resources/model_upload_image_response_attributes.go b/resources/model_upload_image_response_attributes.go index ce847bf..cb04c1f 100644 --- a/resources/model_upload_image_response_attributes.go +++ b/resources/model_upload_image_response_attributes.go @@ -5,6 +5,6 @@ package resources type UploadImageResponseAttributes struct { - // Pre-Signed URL for upload the file + // Pre-signed URL to upload the file Url string `json:"url"` }