From 077199a56ff8003695c5e903814ceb5ab1bea94d Mon Sep 17 00:00:00 2001 From: Konst Date: Wed, 13 Nov 2024 16:20:00 +0300 Subject: [PATCH 1/7] Added rollup stats aggregation API endpoint --- cmd/api/handler/responses/rollup.go | 18 +++++++++++++ cmd/api/handler/rollup.go | 40 +++++++++++++++++++++++++++++ cmd/api/init.go | 1 + internal/storage/postgres/rollup.go | 12 +++++++++ internal/storage/rollup.go | 13 ++++++++++ 5 files changed, 84 insertions(+) diff --git a/cmd/api/handler/responses/rollup.go b/cmd/api/handler/responses/rollup.go index 1dd78f49..cf168375 100644 --- a/cmd/api/handler/responses/rollup.go +++ b/cmd/api/handler/responses/rollup.go @@ -204,3 +204,21 @@ func NewRollupWithDayStats(r storage.RollupWithDayStats) RollupWithDayStats { return response } + +type RollupGroupedStats struct { + Fee float64 `example:"123.456789" format:"string" json:"fee" swaggertype:"string"` + Size float64 `example:"1000" format:"integer" json:"size" swaggertype:"integer"` + BlobsCount float64 `example:"2" format:"integer" json:"blobs_count" swaggertype:"integer"` + Group string `example:"group" format:"string" json:"group" swaggertype:"string"` +} + +func NewRollupGroupedStats(r storage.RollupGroupedStats) RollupGroupedStats { + response := RollupGroupedStats{ + Fee: r.Fee, + Size: r.Size, + BlobsCount: r.BlobsCount, + Group: r.Group, + } + + return response +} diff --git a/cmd/api/handler/rollup.go b/cmd/api/handler/rollup.go index dac9f799..26bb1285 100644 --- a/cmd/api/handler/rollup.go +++ b/cmd/api/handler/rollup.go @@ -537,3 +537,43 @@ func (handler RollupHandler) ExportBlobs(c echo.Context) error { } return nil } + +type rollupGroupStats struct { + Func string `query:"func" validate:"oneof=sum avg"` + Column string `query:"column" validate:"oneof=stack type category vm provider"` +} + +// RollupGroupedStats godoc +// +// @Summary Rollup Grouped Statistics +// @Description Rollup Grouped Statistics +// @Tags rollup +// @ID rollup-grouped-statistics +// @Param func query string false "Aggregate function" Enums(sum, avg) +// @Param column query string false "Group column" Enums(stack, type, category, vm, provider) +// @Produce json +// @Success 200 {array} responses.RollupGroupedStats +// @Failure 400 {object} Error +// @Failure 500 {object} Error +// @Router /rollup/group [get] +func (handler RollupHandler) RollupGroupedStats(c echo.Context) error { + req, err := bindAndValidate[rollupGroupStats](c) + if err != nil { + return badRequestError(c, err) + } + + rollups, err := handler.rollups.RollupStatsGrouping(c.Request().Context(), storage.RollupGroupStatsFilters{ + Func: req.Func, + Column: req.Column, + }) + if err != nil { + return handleError(c, err, handler.rollups) + } + + response := make([]responses.RollupGroupedStats, len(rollups)) + for i := range rollups { + response[i] = responses.NewRollupGroupedStats(rollups[i]) + } + + return returnArray(c, response) +} diff --git a/cmd/api/init.go b/cmd/api/init.go index 9298708f..c8e54fb1 100644 --- a/cmd/api/init.go +++ b/cmd/api/init.go @@ -485,6 +485,7 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto rollups.GET("", rollupHandler.Leaderboard) rollups.GET("/count", rollupHandler.Count) rollups.GET("/day", rollupHandler.LeaderboardDay) + rollups.GET("/group", rollupHandler.RollupGroupedStats) rollups.GET("/stats/series", rollupHandler.AllSeries) rollups.GET("/slug/:slug", rollupHandler.BySlug) rollup := rollups.Group("/:id") diff --git a/internal/storage/postgres/rollup.go b/internal/storage/postgres/rollup.go index 5c735567..7ffaa8f6 100644 --- a/internal/storage/postgres/rollup.go +++ b/internal/storage/postgres/rollup.go @@ -288,3 +288,15 @@ func (r *Rollup) AllSeries(ctx context.Context) (items []storage.RollupHistogram return } + +func (r *Rollup) RollupStatsGrouping(ctx context.Context, fltrs storage.RollupGroupStatsFilters) (results []storage.RollupGroupedStats, err error) { + err = r.DB().NewSelect(). + Table(storage.ViewLeaderboard). + ColumnExpr(fltrs.Func+"(fee) as fee"). + ColumnExpr(fltrs.Func+"(size) as size"). + ColumnExpr(fltrs.Func+"(blobs_count) as blobs_count"). + ColumnExpr(fltrs.Column+" as group"). + Group(fltrs.Column). + Scan(ctx, &results) + return +} diff --git a/internal/storage/rollup.go b/internal/storage/rollup.go index 641fa7ae..12fba0d5 100644 --- a/internal/storage/rollup.go +++ b/internal/storage/rollup.go @@ -21,6 +21,11 @@ type LeaderboardFilters struct { Category []types.RollupCategory } +type RollupGroupStatsFilters struct { + Func string + Column string +} + //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed type IRollup interface { sdk.Table[*Rollup] @@ -36,6 +41,7 @@ type IRollup interface { Count(ctx context.Context) (int64, error) Distribution(ctx context.Context, rollupId uint64, series, groupBy string) (items []DistributionItem, err error) BySlug(ctx context.Context, slug string) (RollupWithStats, error) + RollupStatsGrouping(ctx context.Context, fltrs RollupGroupStatsFilters) ([]RollupGroupedStats, error) } // Rollup - @@ -128,3 +134,10 @@ type RollupHistogramItem struct { Logo string `bun:"logo"` Time time.Time `bun:"time"` } + +type RollupGroupedStats struct { + Fee float64 `bun:"fee"` + Size float64 `bun:"size"` + BlobsCount float64 `bun:"blobs_count"` + Group string `bun:"group"` +} From f1403320e17b44de46e23f60ef213f68b2ecebfc Mon Sep 17 00:00:00 2001 From: Konst Date: Wed, 13 Nov 2024 17:02:35 +0300 Subject: [PATCH 2/7] BlobsCount field type changed to int64 --- cmd/api/handler/responses/rollup.go | 2 +- internal/storage/rollup.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/api/handler/responses/rollup.go b/cmd/api/handler/responses/rollup.go index cf168375..feba0b5b 100644 --- a/cmd/api/handler/responses/rollup.go +++ b/cmd/api/handler/responses/rollup.go @@ -208,7 +208,7 @@ func NewRollupWithDayStats(r storage.RollupWithDayStats) RollupWithDayStats { type RollupGroupedStats struct { Fee float64 `example:"123.456789" format:"string" json:"fee" swaggertype:"string"` Size float64 `example:"1000" format:"integer" json:"size" swaggertype:"integer"` - BlobsCount float64 `example:"2" format:"integer" json:"blobs_count" swaggertype:"integer"` + BlobsCount int64 `example:"2" format:"integer" json:"blobs_count" swaggertype:"integer"` Group string `example:"group" format:"string" json:"group" swaggertype:"string"` } diff --git a/internal/storage/rollup.go b/internal/storage/rollup.go index 12fba0d5..9a7cd6ff 100644 --- a/internal/storage/rollup.go +++ b/internal/storage/rollup.go @@ -138,6 +138,6 @@ type RollupHistogramItem struct { type RollupGroupedStats struct { Fee float64 `bun:"fee"` Size float64 `bun:"size"` - BlobsCount float64 `bun:"blobs_count"` + BlobsCount int64 `bun:"blobs_count"` Group string `bun:"group"` } From 3349be1e6d98dfd53add527d0083cb5fb6a7b2d3 Mon Sep 17 00:00:00 2001 From: Konst Date: Thu, 14 Nov 2024 15:17:58 +0300 Subject: [PATCH 3/7] Added tests --- cmd/api/handler/rollup.go | 2 +- cmd/api/handler/rollup_test.go | 57 +++++++++++++++++++++++++++++++++ internal/storage/mock/rollup.go | 40 +++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/cmd/api/handler/rollup.go b/cmd/api/handler/rollup.go index 26bb1285..69d728a3 100644 --- a/cmd/api/handler/rollup.go +++ b/cmd/api/handler/rollup.go @@ -539,7 +539,7 @@ func (handler RollupHandler) ExportBlobs(c echo.Context) error { } type rollupGroupStats struct { - Func string `query:"func" validate:"oneof=sum avg"` + Func string `query:"func" validate:"oneof=sum avg"` Column string `query:"column" validate:"oneof=stack type category vm provider"` } diff --git a/cmd/api/handler/rollup_test.go b/cmd/api/handler/rollup_test.go index 7fd4cdfb..061fb37f 100644 --- a/cmd/api/handler/rollup_test.go +++ b/cmd/api/handler/rollup_test.go @@ -46,6 +46,12 @@ var ( SizePct: 0.3, }, } + testRollupWithGroupedStats = storage.RollupGroupedStats{ + Fee: 0.1, + Size: 0.2, + BlobsCount: 3, + Group: "stack", + } ) // RollupTestSuite - @@ -461,3 +467,54 @@ func (s *RollupTestSuite) TestAllSeries() { s.Require().EqualValues(1, item.BlobsCount) s.Require().EqualValues(testTime, item.Time) } + +func (s *RollupTestSuite) TestRollupStatsGrouping() { + for _, funcName := range []string{ + "sum", + "avg", + } { + for _, groupName := range []string{ + "stack", + "type", + "category", + "vm", + "provider", + } { + q := make(url.Values) + q.Add("func", funcName) + q.Add("column", groupName) + + req := httptest.NewRequest(http.MethodGet, "/?"+q.Encode(), nil) + rec := httptest.NewRecorder() + c := s.echo.NewContext(req, rec) + c.SetPath("/rollup/group") + + s.rollups.EXPECT(). + RollupStatsGrouping(gomock.Any(), storage.RollupGroupStatsFilters{ + Func: funcName, + Column: groupName, + }). + Return([]storage.RollupGroupedStats{testRollupWithGroupedStats}, nil). + Times(1) + + s.Require().NoError(s.handler.RollupGroupedStats(c)) + s.Require().Equal(http.StatusOK, rec.Code) + var stats []responses.RollupGroupedStats + err := json.NewDecoder(rec.Body).Decode(&stats) + s.Require().NoError(err) + s.Require().Len(stats, 1) + + groupedStats := stats[0] + testRollupWithGroupedStats = storage.RollupGroupedStats{ + Fee: 0.1, + Size: 0.2, + BlobsCount: 3, + Group: "stack", + } + s.Require().EqualValues(0.1, groupedStats.Fee) + s.Require().EqualValues(0.2, groupedStats.Size) + s.Require().EqualValues(3, groupedStats.BlobsCount) + s.Require().EqualValues("stack", groupedStats.Group) + } + } +} diff --git a/internal/storage/mock/rollup.go b/internal/storage/mock/rollup.go index 4d1879d5..a10bc355 100644 --- a/internal/storage/mock/rollup.go +++ b/internal/storage/mock/rollup.go @@ -25,6 +25,7 @@ import ( type MockIRollup struct { ctrl *gomock.Controller recorder *MockIRollupMockRecorder + isgomock struct{} } // MockIRollupMockRecorder is the mock recorder for MockIRollup. @@ -589,6 +590,45 @@ func (c *MockIRollupProvidersCall) DoAndReturn(f func(context.Context, uint64) ( return c } +// RollupStatsGrouping mocks base method. +func (m *MockIRollup) RollupStatsGrouping(ctx context.Context, fltrs storage.RollupGroupStatsFilters) ([]storage.RollupGroupedStats, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollupStatsGrouping", ctx, fltrs) + ret0, _ := ret[0].([]storage.RollupGroupedStats) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RollupStatsGrouping indicates an expected call of RollupStatsGrouping. +func (mr *MockIRollupMockRecorder) RollupStatsGrouping(ctx, fltrs any) *MockIRollupRollupStatsGroupingCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupStatsGrouping", reflect.TypeOf((*MockIRollup)(nil).RollupStatsGrouping), ctx, fltrs) + return &MockIRollupRollupStatsGroupingCall{Call: call} +} + +// MockIRollupRollupStatsGroupingCall wrap *gomock.Call +type MockIRollupRollupStatsGroupingCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockIRollupRollupStatsGroupingCall) Return(arg0 []storage.RollupGroupedStats, arg1 error) *MockIRollupRollupStatsGroupingCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockIRollupRollupStatsGroupingCall) Do(f func(context.Context, storage.RollupGroupStatsFilters) ([]storage.RollupGroupedStats, error)) *MockIRollupRollupStatsGroupingCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockIRollupRollupStatsGroupingCall) DoAndReturn(f func(context.Context, storage.RollupGroupStatsFilters) ([]storage.RollupGroupedStats, error)) *MockIRollupRollupStatsGroupingCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // RollupsByNamespace mocks base method. func (m *MockIRollup) RollupsByNamespace(ctx context.Context, namespaceId uint64, limit, offset int) ([]storage.Rollup, error) { m.ctrl.T.Helper() From 76b571a0076c31d9e2f6440bb89af2e4c8783687 Mon Sep 17 00:00:00 2001 From: Konst Date: Thu, 14 Nov 2024 15:22:10 +0300 Subject: [PATCH 4/7] Added routes tests --- cmd/api/routes_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/api/routes_test.go b/cmd/api/routes_test.go index ed115dcc..fe66325e 100644 --- a/cmd/api/routes_test.go +++ b/cmd/api/routes_test.go @@ -103,6 +103,7 @@ func TestRoutes(t *testing.T) { "/v1/stats/rollup_stats_24h GET": {}, "/v1/stats/messages_count_24h GET": {}, "/v1/rollup/stats/series GET": {}, + "/v1/rollup/group GET": {}, } ctx, cancel := context.WithCancel(context.Background()) From 0d4edf60d334974403eca56d19de65d05857887a Mon Sep 17 00:00:00 2001 From: Konst Date: Thu, 14 Nov 2024 16:22:19 +0300 Subject: [PATCH 5/7] Added validation for DB RollupStatsGrouping method --- internal/storage/postgres/rollup.go | 33 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/internal/storage/postgres/rollup.go b/internal/storage/postgres/rollup.go index 7ffaa8f6..9efdff4e 100644 --- a/internal/storage/postgres/rollup.go +++ b/internal/storage/postgres/rollup.go @@ -290,13 +290,30 @@ func (r *Rollup) AllSeries(ctx context.Context) (items []storage.RollupHistogram } func (r *Rollup) RollupStatsGrouping(ctx context.Context, fltrs storage.RollupGroupStatsFilters) (results []storage.RollupGroupedStats, err error) { - err = r.DB().NewSelect(). - Table(storage.ViewLeaderboard). - ColumnExpr(fltrs.Func+"(fee) as fee"). - ColumnExpr(fltrs.Func+"(size) as size"). - ColumnExpr(fltrs.Func+"(blobs_count) as blobs_count"). - ColumnExpr(fltrs.Column+" as group"). - Group(fltrs.Column). - Scan(ctx, &results) + query := r.DB().NewSelect().Table("leaderboard2") + + switch fltrs.Func { + case "sum": + query = query. + ColumnExpr("sum(fee) as fee"). + ColumnExpr("sum(size) as size"). + ColumnExpr("sum(blobs_count) as blobs_count") + case "avg": + query = query. + ColumnExpr("avg(fee) as fee"). + ColumnExpr("avg(size) as size"). + ColumnExpr("avg(blobs_count) as blobs_count") + default: + return nil, errors.Errorf("unknown func field: %s", fltrs.Column) + } + + switch fltrs.Column { + case "stack", "type", "category", "vm", "provider": + query = query.ColumnExpr(fltrs.Column + " as group").Group(fltrs.Column) + default: + return nil, errors.Errorf("unknown column field: %s", fltrs.Column) + } + + err = query.Scan(ctx, &results) return } From e6829f0889fb8803da2d44ae1d57cc2824a730c9 Mon Sep 17 00:00:00 2001 From: Konst Date: Thu, 14 Nov 2024 16:22:29 +0300 Subject: [PATCH 6/7] Removed unnecessary code in TestRollupStatsGrouping --- cmd/api/handler/rollup_test.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/api/handler/rollup_test.go b/cmd/api/handler/rollup_test.go index 061fb37f..3890d1ed 100644 --- a/cmd/api/handler/rollup_test.go +++ b/cmd/api/handler/rollup_test.go @@ -505,12 +505,7 @@ func (s *RollupTestSuite) TestRollupStatsGrouping() { s.Require().Len(stats, 1) groupedStats := stats[0] - testRollupWithGroupedStats = storage.RollupGroupedStats{ - Fee: 0.1, - Size: 0.2, - BlobsCount: 3, - Group: "stack", - } + s.Require().EqualValues(0.1, groupedStats.Fee) s.Require().EqualValues(0.2, groupedStats.Size) s.Require().EqualValues(3, groupedStats.BlobsCount) From 9fee9a33c2565f5bcdbcdb5aecbee160c769bdff Mon Sep 17 00:00:00 2001 From: Konst Date: Fri, 15 Nov 2024 17:14:04 +0300 Subject: [PATCH 7/7] Added DB tests --- internal/storage/postgres/rollup.go | 2 +- internal/storage/postgres/rollup_test.go | 22 ++++++++++++++++++++++ test/data/rollup.yml | 3 +++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/storage/postgres/rollup.go b/internal/storage/postgres/rollup.go index 9efdff4e..55148f4b 100644 --- a/internal/storage/postgres/rollup.go +++ b/internal/storage/postgres/rollup.go @@ -290,7 +290,7 @@ func (r *Rollup) AllSeries(ctx context.Context) (items []storage.RollupHistogram } func (r *Rollup) RollupStatsGrouping(ctx context.Context, fltrs storage.RollupGroupStatsFilters) (results []storage.RollupGroupedStats, err error) { - query := r.DB().NewSelect().Table("leaderboard2") + query := r.DB().NewSelect().Table(storage.ViewLeaderboard) switch fltrs.Func { case "sum": diff --git a/internal/storage/postgres/rollup_test.go b/internal/storage/postgres/rollup_test.go index f58da628..82996de7 100644 --- a/internal/storage/postgres/rollup_test.go +++ b/internal/storage/postgres/rollup_test.go @@ -225,3 +225,25 @@ func (s *StorageTestSuite) TestRollupAllSeries() { s.Require().NoError(err) s.Require().Len(items, 5) } + +func (s *StorageTestSuite) TestRollupStatsGrouping() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + _, err := s.storage.Connection().Exec(ctx, "REFRESH MATERIALIZED VIEW leaderboard;") + s.Require().NoError(err) + + column := "stack" + rollups, err := s.storage.Rollup.RollupStatsGrouping(ctx, storage.RollupGroupStatsFilters{ + Func: "sum", + Column: column, + }) + s.Require().NoError(err, column) + s.Require().Len(rollups, 2, column) + + rollup := rollups[1] + s.Require().EqualValues(4000, rollup.Fee, column) + s.Require().EqualValues(52, rollup.Size, column) + s.Require().EqualValues(4, rollup.BlobsCount, column) + s.Require().EqualValues("OP Stack", rollup.Group, column) +} diff --git a/test/data/rollup.yml b/test/data/rollup.yml index 530ee2d3..14052a2f 100644 --- a/test/data/rollup.yml +++ b/test/data/rollup.yml @@ -7,6 +7,7 @@ logo: https://rollup1.com/image.png slug: rollup_1 category: finance + stack: OP Stack - id: 2 name: Rollup 2 description: The second @@ -16,6 +17,7 @@ logo: https://rollup2.com/image.png slug: rollup_2 category: gaming + stack: OP Stack - id: 3 name: Rollup 3 description: The third @@ -25,3 +27,4 @@ logo: https://rollup3.com/image.png slug: rollup_3 category: nft + stack: Custom Stack