From 8c6b1579e235d93da64426fc75174e47cd38a5d3 Mon Sep 17 00:00:00 2001 From: Stanislav Viyachev <66465846+Coiling-Dragon@users.noreply.github.com> Date: Thu, 25 May 2023 17:00:23 +0300 Subject: [PATCH] Add timestamp query params (#895) * Add timestamp query params * Fix transaction history count query * Remove transaction count function * update docs --------- Signed-off-by: Stanislav Viyachev --- app/domain/repository/transfer.go | 3 +- app/domain/service/errors.go | 1 + app/helper/http/error-response.go | 3 + app/model/transfer/transfer.go | 10 +- app/persistence/transfer/transfer.go | 106 +++++++++++----- app/persistence/transfer/transfer_test.go | 118 ++++++++++++------ app/services/transfers/service.go | 8 +- docs/api.md | 3 +- examples/three-validators/docker-compose.yml | 4 +- .../repository/transfer_repository_mock.go | 6 +- 10 files changed, 172 insertions(+), 90 deletions(-) diff --git a/app/domain/repository/transfer.go b/app/domain/repository/transfer.go index 4b9343e90..50d6443d4 100644 --- a/app/domain/repository/transfer.go +++ b/app/domain/repository/transfer.go @@ -33,6 +33,5 @@ type Transfer interface { Create(ct *payload.Transfer) (*entity.Transfer, error) UpdateStatusCompleted(txId string) error UpdateStatusFailed(txId string) error - Paged(req *transfer.PagedRequest) ([]*entity.Transfer, error) - Count() (int64, error) + Paged(req *transfer.PagedRequest) ([]*entity.Transfer, int64, error) } diff --git a/app/domain/service/errors.go b/app/domain/service/errors.go index dfc66ed18..4c6300ce1 100644 --- a/app/domain/service/errors.go +++ b/app/domain/service/errors.go @@ -20,3 +20,4 @@ import "errors" var ErrNotFound = errors.New("not found") var ErrBadRequestTransferTargetNetworkNoSignaturesRequired = errors.New("transfer target network does not require signatures") +var ErrWrongQuery = errors.New("wrong query parameter") diff --git a/app/helper/http/error-response.go b/app/helper/http/error-response.go index 6cf8b7dbe..67d8c952a 100644 --- a/app/helper/http/error-response.go +++ b/app/helper/http/error-response.go @@ -32,6 +32,9 @@ func WriteErrorResponse(w http.ResponseWriter, r *http.Request, err error) { case service.ErrNotFound: render.Status(r, http.StatusNotFound) render.JSON(w, r, response.ErrorResponse(err)) + case service.ErrWrongQuery: + render.Status(r, http.StatusBadRequest) + render.JSON(w, r, response.ErrorResponse(err)) default: render.Status(r, http.StatusInternalServerError) render.JSON(w, r, response.ErrorResponse(response.ErrorInternalServerError)) diff --git a/app/model/transfer/transfer.go b/app/model/transfer/transfer.go index 8921347e2..141f79c26 100644 --- a/app/model/transfer/transfer.go +++ b/app/model/transfer/transfer.go @@ -54,10 +54,10 @@ type PagedRequest struct { } type Filter struct { - Originator string `json:"originator"` - Timestamp time.Time `json:"timestamp"` - TokenId string `json:"tokenId"` - TransactionId string `json:"transactionId"` + Originator string `json:"originator"` + TimestampQuery string `json:"timestamp"` + TokenId string `json:"tokenId"` + TransactionId string `json:"transactionId"` } type SanityCheckResult struct { @@ -72,5 +72,5 @@ type TransferReset struct { SourceChainId uint64 `json:"sourceChainId"` TargetChainId uint64 `json:"targetChainId"` SourceToken string `json:"sourceToken"` - Password string `json:"password"` + Password string `json:"password"` } diff --git a/app/persistence/transfer/transfer.go b/app/persistence/transfer/transfer.go index 0005421fb..ed0937f3d 100644 --- a/app/persistence/transfer/transfer.go +++ b/app/persistence/transfer/transfer.go @@ -20,10 +20,13 @@ import ( "database/sql" "errors" "fmt" - "github.com/limechain/hedera-eth-bridge-validator/constants" "strings" + "time" + + "github.com/limechain/hedera-eth-bridge-validator/constants" "github.com/ethereum/go-ethereum/common" + "github.com/limechain/hedera-eth-bridge-validator/app/domain/service" "github.com/limechain/hedera-eth-bridge-validator/app/model/transfer" "github.com/limechain/hedera-eth-bridge-validator/app/process/payload" @@ -135,15 +138,66 @@ func (r *Repository) UpdateStatusFailed(txId string) error { return r.updateStatus(txId, status.Failed) } -func (r *Repository) Paged(req *transfer.PagedRequest) ([]*entity.Transfer, error) { +func formatTimestampFilter(q *gorm.DB, ts_query string) (*gorm.DB, error) { + qParams := strings.Split(ts_query, "&") + operators := map[string]string{ + "eq": "=", + "gt": ">", + "lt": "<", + "gte": ">=", + "lte": "<=", + } + + // This if statement handles legacy timestamp filter like: + // "timestamp": "2023-04-19T04:41:47.104114905Z" + if len(qParams) == 1 && !strings.Contains(ts_query, "=") { + timestamp, err := time.Parse(time.RFC3339Nano, qParams[0]) + if err != nil { + return q, err + } + + q = q.Where("timestamp = ?", timestamp.UnixNano()) + return q, nil + } + + // This for loop handles new timestamp filter like: + // "timestamp": "lte=2023-05-19T04:41:47.104114905Z>e=2023-04-19T04:41:47.104114905Z" + // "timestamp": "eq=2023-04-19T04:41:47.104114905Z" + for _, param := range qParams { + parts := strings.Split(param, "=") + + operator := parts[0] + datetime, err := time.Parse(time.RFC3339Nano, parts[1]) + if err != nil { + return q, err + } + + timestamp := datetime.UnixNano() + + op, exists := operators[operator] + if !exists { + return q, service.ErrWrongQuery + } + + q = q.Where("timestamp "+op+" ?", timestamp) + + } + + return q, nil +} + +func (r *Repository) Paged(req *transfer.PagedRequest) ([]*entity.Transfer, int64, error) { + var ( + err error + count int64 + ) + offset := (req.Page - 1) * req.PageSize res := make([]*entity.Transfer, 0, req.PageSize) f := req.Filter q := r.db. Model(entity.Transfer{}). - Order("timestamp desc, status asc"). - Offset(int(offset)). - Limit(int(req.PageSize)) + Order("timestamp desc, status asc") if f.Originator != "" { if strings.Contains(f.Originator, "0x") { @@ -153,8 +207,14 @@ func (r *Repository) Paged(req *transfer.PagedRequest) ([]*entity.Transfer, erro q = q.Where("originator = ?", f.Originator) } } - if !f.Timestamp.IsZero() { - q = q.Where("timestamp = ?", f.Timestamp.UnixNano()) + + if f.TimestampQuery != "" { + q, err = formatTimestampFilter(q, f.TimestampQuery) + if err != nil { + r.logger.Errorf("Failed to get paged transfers: [%s]", err) + return nil, 0, err + } + } if f.TokenId != "" { if strings.Contains(f.TokenId, "0x") { @@ -168,35 +228,17 @@ func (r *Repository) Paged(req *transfer.PagedRequest) ([]*entity.Transfer, erro q = q.Where("transaction_id LIKE ?", fmt.Sprintf(`%s%%`, f.TransactionId)) } - err := q.Find(&res).Error - if err != nil { - r.logger.Errorf("Failed to get paged transfers: [%s]", err) - return nil, err - } - - return res, nil -} - -func (r *Repository) Count() (int64, error) { - db, err := r.db.DB() - if err != nil { - return 0, err - } + q = q.Count(&count). + Offset(int(offset)). + Limit(int(req.PageSize)) - cur, err := db.Query(`SELECT COUNT(*) FROM (SELECT DISTINCT transaction_id FROM transfers) AS t`) + err = q.Find(&res).Error if err != nil { - return 0, err - } - defer cur.Close() - - var res int64 - if cur.Next() { - if err := cur.Scan(&res); err != nil { - return 0, err - } + r.logger.Errorf("Failed to get paged transfers: [%s]", err) + return nil, 0, err } - return res, nil + return res, count, nil } func (r *Repository) create(ct *payload.Transfer, status string) (*entity.Transfer, error) { diff --git a/app/persistence/transfer/transfer_test.go b/app/persistence/transfer/transfer_test.go index 5a316ae22..658c499d2 100644 --- a/app/persistence/transfer/transfer_test.go +++ b/app/persistence/transfer/transfer_test.go @@ -19,11 +19,13 @@ package transfer import ( "database/sql" "database/sql/driver" - "github.com/limechain/hedera-eth-bridge-validator/constants" + "fmt" "regexp" "testing" "time" + "github.com/limechain/hedera-eth-bridge-validator/constants" + "github.com/ethereum/go-ethereum/common" "github.com/limechain/hedera-eth-bridge-validator/app/model/transfer" @@ -188,12 +190,14 @@ var ( updateFeeQuery = regexp.QuoteMeta(`UPDATE "transfers" SET "fee"=$1 WHERE transaction_id = $2`) updateStatusQuery = regexp.QuoteMeta(`UPDATE "transfers" SET "status"=$1 WHERE transaction_id = $2`) - countQuery = regexp.QuoteMeta(`SELECT COUNT(*) FROM (SELECT DISTINCT transaction_id FROM transfers) AS t`) - pagedQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" ORDER BY timestamp desc, status asc LIMIT 10 OFFSET 10`) - pagedFilterOriginatorQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE originator = $1 ORDER BY timestamp desc, status asc LIMIT 10`) - pagedFilterTimestampQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE timestamp = $1 ORDER BY timestamp desc, status asc LIMIT 10`) - pagedFilterTransactionIdQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE transaction_id LIKE $1 ORDER BY timestamp desc, status asc LIMIT 10`) - pagedFilterTokenIdQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE (source_asset = $1 OR target_asset = $2) ORDER BY timestamp desc, status asc LIMIT 10`) + // "SELECT count(*) FROM \"transfers\"\" + countQuery = regexp.QuoteMeta(`SELECT count(*) FROM "transfers"`) + pagedQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" ORDER BY timestamp desc, status asc LIMIT 10 OFFSET 10`) + pagedFilterOriginatorQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE originator = $1 ORDER BY timestamp desc, status asc LIMIT 10`) + pagedFilterTimestampQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE timestamp = $1 ORDER BY timestamp desc, status asc LIMIT 10`) + pagedFilterFromToTimestampQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE timestamp <= $1 AND timestamp >= $2 ORDER BY timestamp desc, status asc LIMIT 10`) + pagedFilterTransactionIdQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE transaction_id LIKE $1 ORDER BY timestamp desc, status asc LIMIT 10`) + pagedFilterTokenIdQuery = regexp.QuoteMeta(`SELECT * FROM "transfers" WHERE (source_asset = $1 OR target_asset = $2) ORDER BY timestamp desc, status asc LIMIT 10`) ) func setup() { @@ -554,9 +558,13 @@ func Test_Paged(t *testing.T) { Page: 2, PageSize: 10, } + + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedQuery) - actual, err := repository.Paged(req) + actual, _, err := repository.Paged(req) assert.Nil(t, err) assert.NotEmpty(t, actual) @@ -569,9 +577,13 @@ func Test_PagedWithErr(t *testing.T) { Page: 2, PageSize: 10, } + + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + _ = helper.SqlMockPrepareQueryWithErrInvalidData(sqlMock, pagedQuery) - actual, err := repository.Paged(req) + actual, _, err := repository.Paged(req) assert.NotNil(t, err) assert.Empty(t, actual) @@ -587,9 +599,13 @@ func Test_PagedWithFilterOriginatorHedera(t *testing.T) { Originator: originator, }, } + + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterOriginatorQuery, originator) - actual, err := repository.Paged(req) + actual, _, err := repository.Paged(req) assert.Nil(t, err) assert.NotEmpty(t, actual) @@ -605,87 +621,117 @@ func Test_PagedWithFilterOriginatorEVM(t *testing.T) { Originator: originatorEVM, }, } + + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterOriginatorQuery, common.HexToAddress(originatorEVM).String()) - actual, err := repository.Paged(req) + actual, _, err := repository.Paged(req) assert.Nil(t, err) assert.NotEmpty(t, actual) } -func Test_PagedWithFilterTimestamp(t *testing.T) { +func Test_PagedWithFilterLegacyTimestamp(t *testing.T) { setup() defer helper.CheckSqlMockExpectationsMet(sqlMock, t) + string_time := nanoTime.Time.Format(time.RFC3339Nano) req := &transfer.PagedRequest{ Page: 1, PageSize: 10, Filter: transfer.Filter{ - Timestamp: nanoTime.Time, + TimestampQuery: string_time, }, } + + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterTimestampQuery, nanoTime.Time.UnixNano()) - actual, err := repository.Paged(req) + actual, _, err := repository.Paged(req) assert.Nil(t, err) assert.NotEmpty(t, actual) } -func Test_PagedWithFilterTransactionId(t *testing.T) { +func Test_PagedWithFilterTimestamp(t *testing.T) { setup() defer helper.CheckSqlMockExpectationsMet(sqlMock, t) + + to_string_time := nanoTime.Time.Format(time.RFC3339Nano) + from_string_time := nanoTime.Time.Add(-time.Hour).Format(time.RFC3339Nano) + formated_time_string := fmt.Sprintf("lte=%s>e=%s", to_string_time, from_string_time) + req := &transfer.PagedRequest{ Page: 1, PageSize: 10, Filter: transfer.Filter{ - TransactionId: transactionId, + TimestampQuery: formated_time_string, }, } - helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterTransactionIdQuery, txIdWithPlaceholder) - actual, err := repository.Paged(req) + // Parse the formatted timestamp strings + to_timestamp, err := time.Parse(time.RFC3339Nano, to_string_time) + if err != nil { + panic(err) + } + from_timestamp, err := time.Parse(time.RFC3339Nano, from_string_time) + if err != nil { + panic(err) + } + + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterFromToTimestampQuery, to_timestamp.UnixNano(), from_timestamp.UnixNano()) + actual, _, _ := repository.Paged(req) + fmt.Println(actual) assert.Nil(t, err) assert.NotEmpty(t, actual) } -func Test_PagedWithFilterTokenId(t *testing.T) { +func Test_PagedWithFilterTransactionId(t *testing.T) { setup() defer helper.CheckSqlMockExpectationsMet(sqlMock, t) req := &transfer.PagedRequest{ Page: 1, PageSize: 10, Filter: transfer.Filter{ - TokenId: sourceAsset, + TransactionId: transactionId, }, } - helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterTokenIdQuery, sourceAsset, sourceAsset) - actual, err := repository.Paged(req) + expected := int64(1) + helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterTransactionIdQuery, txIdWithPlaceholder) + + actual, _, err := repository.Paged(req) assert.Nil(t, err) assert.NotEmpty(t, actual) } -func Test_Count(t *testing.T) { +func Test_PagedWithFilterTokenId(t *testing.T) { setup() defer helper.CheckSqlMockExpectationsMet(sqlMock, t) + req := &transfer.PagedRequest{ + Page: 1, + PageSize: 10, + Filter: transfer.Filter{ + TokenId: sourceAsset, + }, + } + expected := int64(1) helper.SqlMockPrepareQuery(sqlMock, []string{"count"}, []driver.Value{expected}, countQuery) + helper.SqlMockPrepareQuery(sqlMock, transferColumns, transferRowArgs, pagedFilterTokenIdQuery, sourceAsset, sourceAsset) - actual, err := repository.Count() + actual, _, err := repository.Paged(req) assert.Nil(t, err) - assert.Equal(t, expected, actual) -} - -func Test_CountWithErr(t *testing.T) { - setup() - defer helper.CheckSqlMockExpectationsMet(sqlMock, t) - _ = helper.SqlMockPrepareQueryWithErrInvalidData(sqlMock, countQuery) - - actual, err := repository.Count() - - assert.NotNil(t, err) - assert.Equal(t, int64(0), actual) + assert.NotEmpty(t, actual) } diff --git a/app/services/transfers/service.go b/app/services/transfers/service.go index d3456af2e..3d843a6ae 100644 --- a/app/services/transfers/service.go +++ b/app/services/transfers/service.go @@ -375,7 +375,7 @@ func (ts *Service) TransferData(txId string) (interface{}, error) { } func (ts *Service) Paged(req *model.PagedRequest) (*model.Paged, error) { - items, err := ts.transferRepository.Paged(req) + items, count, err := ts.transferRepository.Paged(req) if err != nil { ts.logger.Errorf("Failed to get paged transfers. Error: [%s]", err) return nil, err @@ -386,12 +386,6 @@ func (ts *Service) Paged(req *model.PagedRequest) (*model.Paged, error) { res = append(res, t.ToDto()) } - count, err := ts.transferRepository.Count() - if err != nil { - ts.logger.Errorf("Failed to count transfers. Error: [%s]", err) - return nil, err - } - return &model.Paged{ Items: res, TotalCount: count, diff --git a/docs/api.md b/docs/api.md index 1aead70c4..2eefc441e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -30,13 +30,14 @@ Example: ``` - `POST /api/v1/transfers/history`: Accepts a request body in the form (`*` is required) and returns: - Maximum page size is 50. Pages start from 1. + - Parameter timestamp supports query params like `gt`, `lt`, `gte`, `lte`, `eq` to filter by range. - ```json { *"page": 1, *"pageSize": 20, "filter": { "originator": "Hedera account ID or EVM address", - "timestamp": "VALID RFC3339(Nano) DATE", + "timestamp": "VALID RFC3339(Nano) DATE. Supports query params. Ex: 2021-08-31T00:00:00.000000000Z. Ex-2: gte=2023-05-25T07:43:08.650830003Z<e=2023-05-25T08:11:10.058833356Z", "tokenId": "Hedera Token ID or EVM address", "transactionId": "Hedera Transaction ID or EVM transaction hash" } diff --git a/examples/three-validators/docker-compose.yml b/examples/three-validators/docker-compose.yml index cb10e3cd1..ef4db89c8 100644 --- a/examples/three-validators/docker-compose.yml +++ b/examples/three-validators/docker-compose.yml @@ -80,7 +80,7 @@ services: bob: image: eth-hedera-validator environment: - VTAG: ${TAG} + VERSION_TAG: ${TAG} VALIDATOR_DATABASE_HOST: bob_db volumes: - ./bridge.yml:/src/hedera-eth-bridge-validator/config/bridge.yml @@ -94,7 +94,7 @@ services: carol: image: eth-hedera-validator environment: - VTAG: ${TAG} + VERSION_TAG: ${TAG} VALIDATOR_DATABASE_HOST: carol_db volumes: - ./bridge.yml:/src/hedera-eth-bridge-validator/config/bridge.yml diff --git a/test/mocks/repository/transfer_repository_mock.go b/test/mocks/repository/transfer_repository_mock.go index 23b2d857d..1daa776de 100644 --- a/test/mocks/repository/transfer_repository_mock.go +++ b/test/mocks/repository/transfer_repository_mock.go @@ -84,10 +84,6 @@ func (m *MockTransferRepository) UpdateStatusCompleted(txId string) error { return args.Get(0).(error) } -func (m *MockTransferRepository) Paged(req *transfer.PagedRequest) ([]*entity.Transfer, error) { - panic("implement me") -} - -func (m *MockTransferRepository) Count() (int64, error) { +func (m *MockTransferRepository) Paged(req *transfer.PagedRequest) ([]*entity.Transfer, int64, error) { panic("implement me") }