Add spreadsheets report generator

config.yaml

@@ -18,9 +18,16 @@ forms:
 
 storage:
   backend: "do"
-  endpoint: https://fra1.digitaloceanspaces.com
+  endpoint: https://nyc3.digitaloceanspaces.com
   bucket: bucket
   presigned_url_expiration: 3m
 
+spreadsheets:
+  credentials: "./credentials.json"
+  token: "./token.json"
+  period: 1m
+  min_abnormal_period: 1m
+  max_abnormal_period: 10m
+
 auth:
   addr: http://127.0.0.1:5000

go.mod

@@ -23,6 +23,9 @@ require (
 )
 
 require (
+	cloud.google.com/go/auth v0.7.2 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
+	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/Workiva/go-datastructures v1.0.53 // indirect
@@ -43,13 +46,21 @@ require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/ethereum/go-ethereum v1.13.8 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/getsentry/raven-go v0.2.0 // indirect
 	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/jsonapi v0.0.0-20200226002910-c8283f632fb7 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.5 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
@@ -83,15 +94,25 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	gitlab.com/distributed_lab/lorem v0.2.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/api v0.189.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect
+	google.golang.org/grpc v1.64.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect

internal/cli/main.go

@@ -10,7 +10,7 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/rarimo/geo-forms-svc/internal/config"
 	"github.com/rarimo/geo-forms-svc/internal/service"
-	"github.com/rarimo/geo-forms-svc/internal/service/workers/formsender"
+	"github.com/rarimo/geo-forms-svc/internal/service/workers/spreadsheets"
 	"gitlab.com/distributed_lab/kit/kv"
 	"gitlab.com/distributed_lab/logan/v3"
 )
@@ -59,7 +59,12 @@ func Run(args []string) bool {
 	switch cmd {
 	case serviceCmd.FullCommand():
 		run(service.Run)
-		run(formsender.Run)
+
+		wg.Add(1)
+		go func() {
+			spreadsheets.Run(ctx, cfg)
+			wg.Done()
+		}()
 	case migrateUpCmd.FullCommand():
 		err = MigrateUp(cfg)
 	case migrateDownCmd.FullCommand():

internal/config/forms.go

@@ -41,18 +41,17 @@ func (c *config) Forms() *Forms {
 			panic(fmt.Errorf("failed to figure out withdrawal point price: %w", err))
 		}
 
-		db, err := sql.Open("mysql", cfg.URL)
-		if err != nil {
-			panic(fmt.Errorf("failed to connect to mysql: %w", err))
-		}
+		// db, err := sql.Open("mysql", cfg.URL)
+		// if err != nil {
+		// 	panic(fmt.Errorf("failed to connect to mysql: %w", err))
+		// }
 
 		return &Forms{
 			Cooldown:          cfg.Cooldown,
 			Period:            cfg.Period,
 			MinAbnormalPeriod: cfg.MinAbnormalPeriod,
 			MaxAbnormalPeriod: cfg.MaxAbnormalPeriod,
 			ResendFormsCount:  cfg.ResendFormsCount,
-			db:                db,
 		}
 	}).(*Forms)
 }

internal/config/main.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"github.com/rarimo/geo-auth-svc/pkg/auth"
+	"github.com/rarimo/geo-forms-svc/internal/service/workers/spreadsheets"
 	"github.com/rarimo/geo-forms-svc/internal/storage"
 	"gitlab.com/distributed_lab/kit/comfig"
 	"gitlab.com/distributed_lab/kit/kv"
@@ -14,6 +15,7 @@ type Config interface {
 	comfig.Listenerer
 	auth.Auther
 	storage.Storager
+	spreadsheets.Spreadsheeter
 
 	Forms() *Forms
 }
@@ -24,6 +26,7 @@ type config struct {
 	comfig.Listenerer
 	auth.Auther
 	storage.Storager
+	spreadsheets.Spreadsheeter
 
 	forms comfig.Once
 
@@ -32,11 +35,12 @@ type config struct {
 
 func New(getter kv.Getter) Config {
 	return &config{
-		getter:     getter,
-		Databaser:  pgdb.NewDatabaser(getter),
-		Listenerer: comfig.NewListenerer(getter),
-		Logger:     comfig.NewLogger(getter, comfig.LoggerOpts{}),
-		Auther:     auth.NewAuther(getter),
-		Storager:   storage.NewStorager(getter),
+		getter:        getter,
+		Databaser:     pgdb.NewDatabaser(getter),
+		Listenerer:    comfig.NewListenerer(getter),
+		Logger:        comfig.NewLogger(getter, comfig.LoggerOpts{}),
+		Auther:        auth.NewAuther(getter),
+		Storager:      storage.NewStorager(getter),
+		Spreadsheeter: spreadsheets.NewSpreadsheeter(getter),
 	}
 }

internal/data/forms.go

@@ -75,4 +75,6 @@ type FormsQ interface {
 
 	FilterByID(ids ...string) FormsQ
 	FilterByStatus(status ...string) FormsQ
+	FilterByUpdatedAt(time.Time) FormsQ
+	FilterImages() FormsQ
 }

internal/data/pg/forms.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/Masterminds/squirrel"
 	"github.com/rarimo/geo-forms-svc/internal/data"
@@ -134,6 +135,14 @@ func (q *formsQ) FilterByStatus(status ...string) data.FormsQ {
 	return q.applyCondition(squirrel.Eq{"status": status})
 }
 
+func (q *formsQ) FilterByUpdatedAt(times time.Time) data.FormsQ {
+	return q.applyCondition(squirrel.Gt{"updated_at": times})
+}
+
+func (q *formsQ) FilterImages() data.FormsQ {
+	return q.applyCondition(squirrel.Eq{"image": nil})
+}
+
 func (q *formsQ) applyCondition(cond squirrel.Sqlizer) data.FormsQ {
 	q.selector = q.selector.Where(cond)
 	q.last = q.selector.Where(cond)

internal/service/handlers/legacy_submit_form.go

@@ -45,7 +45,7 @@ func LegacySubmitForm(w http.ResponseWriter, r *http.Request) {
 	userData := req.Data.Attributes
 	form := &data.Form{
 		Nullifier: nullifier,
-		Status:    data.ProcessedStatus,
+		Status:    data.AcceptedStatus,
 		Name:      userData.Name,
 		Surname:   userData.Surname,
 		IDNum:     userData.IdNum,
@@ -62,11 +62,6 @@ func LegacySubmitForm(w http.ResponseWriter, r *http.Request) {
 		Image:     &userData.Image,
 	}
 
-	if err = Forms(r).SendForms(form); err != nil {
-		Log(r).WithError(err).Error("Failed to send form")
-		form.Status = data.AcceptedStatus
-	}
-
 	_, err = FormsQ(r).Insert(form)
 	if err != nil {
 		Log(r).WithError(err).Error("Failed to insert form")

internal/service/requests/upload_image.go

@@ -8,7 +8,7 @@ import (
 	"github.com/rarimo/geo-forms-svc/resources"
 )
 
-const maxImageSize = 1 << 22
+const maxImageSize = int64(1 << 22)
 
 func NewUploadImage(r *http.Request) (req resources.UploadImageRequest, err error) {
 	if err = json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -19,7 +19,7 @@ func NewUploadImage(r *http.Request) (req resources.UploadImageRequest, err erro
 	errs := validation.Errors{
 		"data/type":                      validation.Validate(req.Data.Type, validation.Required, validation.In(resources.UPLOAD_IMAGE)),
 		"data/attributes/content_type":   validation.Validate(req.Data.Attributes.ContentType, validation.Required, validation.In("image/png", "image/jpeg")),
-		"data/attributes/content_length": validation.Validate(req.Data.Attributes.ContentLength, validation.Required, validation.Length(1, int(maxImageSize))),
+		"data/attributes/content_length": validation.Validate(req.Data.Attributes.ContentLength, validation.Required, validation.Min(int64(1)), validation.Max(maxImageSize)),
 	}
 
 	return req, errs.Filter()