From 632ee1fb7fdbc62305209da6cec953d3e8ece847 Mon Sep 17 00:00:00 2001 From: Jason Brill Date: Tue, 5 Nov 2024 13:47:51 -0500 Subject: [PATCH] tapdb: universe query rewrites --- tapdb/sqlc/querier.go | 2 - tapdb/sqlc/queries/universe.sql | 113 ++++++++------------ tapdb/sqlc/universe.sql.go | 121 ++++++++------------- tapdb/universe_perf_test.go | 182 ++++++++++++++++---------------- 4 files changed, 183 insertions(+), 235 deletions(-) diff --git a/tapdb/sqlc/querier.go b/tapdb/sqlc/querier.go index 62c213c34..04daa9a6b 100644 --- a/tapdb/sqlc/querier.go +++ b/tapdb/sqlc/querier.go @@ -131,8 +131,6 @@ type Querier interface { QueryAssets(ctx context.Context, arg QueryAssetsParams) ([]QueryAssetsRow, error) QueryEventIDs(ctx context.Context, arg QueryEventIDsParams) ([]QueryEventIDsRow, error) QueryFederationGlobalSyncConfigs(ctx context.Context) ([]FederationGlobalSyncConfig, error) - // Join on mssmt_nodes to get leaf related fields. - // Join on genesis_info_view to get leaf related fields. QueryFederationProofSyncLog(ctx context.Context, arg QueryFederationProofSyncLogParams) ([]QueryFederationProofSyncLogRow, error) QueryFederationUniSyncConfigs(ctx context.Context) ([]FederationUniSyncConfig, error) QueryMultiverseLeaves(ctx context.Context, arg QueryMultiverseLeavesParams) ([]QueryMultiverseLeavesRow, error) diff --git a/tapdb/sqlc/queries/universe.sql b/tapdb/sqlc/queries/universe.sql index 787162906..53e3a3550 100644 --- a/tapdb/sqlc/queries/universe.sql +++ b/tapdb/sqlc/queries/universe.sql @@ -6,10 +6,10 @@ FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets - ON genesis_assets.asset_id = universe_roots.asset_id + ON genesis_assets.asset_id = universe_roots.asset_id WHERE mssmt_nodes.namespace = @namespace; -- name: UpsertUniverseRoot :one @@ -30,7 +30,7 @@ WITH root_id AS ( WHERE namespace_root = @namespace_root ) DELETE FROM universe_events -WHERE universe_root_id = (SELECT id from root_id); +WHERE universe_root_id = (SELECT id FROM root_id); -- name: DeleteUniverseRoot :exec DELETE FROM universe_roots @@ -54,25 +54,23 @@ DELETE FROM universe_leaves WHERE leaf_node_namespace = @namespace; -- name: QueryUniverseLeaves :many -SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value genesis_proof, - nodes.sum sum_amt, gen.asset_id -FROM universe_leaves leaves -JOIN mssmt_nodes nodes - ON leaves.leaf_node_key = nodes.key AND - leaves.leaf_node_namespace = nodes.namespace -JOIN genesis_info_view gen +SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value AS genesis_proof, + nodes.sum AS sum_amt, gen.asset_id +FROM universe_leaves AS leaves +JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace +JOIN genesis_info_view AS gen ON leaves.asset_genesis_id = gen.gen_asset_id WHERE leaves.leaf_node_namespace = @namespace - AND - (leaves.minting_point = sqlc.narg('minting_point_bytes') OR - sqlc.narg('minting_point_bytes') IS NULL) - AND - (leaves.script_key_bytes = sqlc.narg('script_key_bytes') OR - sqlc.narg('script_key_bytes') IS NULL); + AND (leaves.minting_point = sqlc.narg('minting_point_bytes') OR + sqlc.narg('minting_point_bytes') IS NULL) + AND (leaves.script_key_bytes = sqlc.narg('script_key_bytes') OR + sqlc.narg('script_key_bytes') IS NULL); -- name: FetchUniverseKeys :many SELECT leaves.minting_point, leaves.script_key_bytes -FROM universe_leaves leaves +FROM universe_leaves AS leaves WHERE leaves.leaf_node_namespace = @namespace ORDER BY CASE WHEN sqlc.narg('sort_direction') = 0 THEN leaves.id END ASC, @@ -84,14 +82,14 @@ SELECT * FROM universe_leaves; -- name: UniverseRoots :many SELECT universe_roots.asset_id, group_key, proof_type, - mssmt_roots.root_hash root_hash, mssmt_nodes.sum root_sum, - genesis_assets.asset_tag asset_name + mssmt_roots.root_hash AS root_hash, mssmt_nodes.sum AS root_sum, + genesis_assets.asset_tag AS asset_name FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets ON genesis_assets.asset_id = universe_roots.asset_id ORDER BY @@ -329,8 +327,8 @@ SELECT SUM(CASE WHEN event_type = 'SYNC' THEN 1 ELSE 0 END) AS sync_events, SUM(CASE WHEN event_type = 'NEW_PROOF' THEN 1 ELSE 0 END) AS new_proof_events FROM universe_events -WHERE event_type IN ('SYNC', 'NEW_PROOF') AND - event_timestamp >= @start_time AND event_timestamp <= @end_time +WHERE event_type IN ('SYNC', 'NEW_PROOF') + AND event_timestamp BETWEEN @start_time AND @end_time GROUP BY day ORDER BY day; @@ -367,7 +365,7 @@ FROM federation_uni_sync_config ORDER BY group_key NULLS LAST, asset_id NULLS LAST, proof_type; -- name: UpsertFederationProofSyncLog :one -INSERT INTO federation_proof_sync_log as log ( +INSERT INTO federation_proof_sync_log AS log ( status, timestamp, sync_direction, proof_leaf_id, universe_root_id, servers_id ) VALUES ( @@ -401,7 +399,7 @@ DO UPDATE SET timestamp = EXCLUDED.timestamp, -- Increment the attempt counter. attempt_counter = CASE - WHEN @bump_sync_attempt_counter = true THEN log.attempt_counter + 1 + WHEN @bump_sync_attempt_counter = TRUE THEN log.attempt_counter + 1 ELSE log.attempt_counter END RETURNING id; @@ -409,58 +407,39 @@ RETURNING id; -- name: QueryFederationProofSyncLog :many SELECT log.id, status, timestamp, sync_direction, attempt_counter, - -- Select fields from the universe_servers table. - server.id as server_id, + server.id AS server_id, server.server_host, - -- Select universe leaf related fields. - leaf.minting_point as leaf_minting_point_bytes, - leaf.script_key_bytes as leaf_script_key_bytes, - mssmt_node.value as leaf_genesis_proof, - genesis.gen_asset_id as leaf_gen_asset_id, - genesis.asset_id as leaf_asset_id, - + leaf.minting_point AS leaf_minting_point_bytes, + leaf.script_key_bytes AS leaf_script_key_bytes, + mssmt_node.value AS leaf_genesis_proof, + genesis.gen_asset_id AS leaf_gen_asset_id, + genesis.asset_id AS leaf_asset_id, -- Select fields from the universe_roots table. - root.asset_id as uni_asset_id, - root.group_key as uni_group_key, - root.proof_type as uni_proof_type - -FROM federation_proof_sync_log as log - -JOIN universe_leaves as leaf + root.asset_id AS uni_asset_id, + root.group_key AS uni_group_key, + root.proof_type AS uni_proof_type +FROM federation_proof_sync_log AS log +JOIN universe_leaves AS leaf ON leaf.id = log.proof_leaf_id - -- Join on mssmt_nodes to get leaf related fields. -JOIN mssmt_nodes mssmt_node - ON leaf.leaf_node_key = mssmt_node.key AND - leaf.leaf_node_namespace = mssmt_node.namespace - +JOIN mssmt_nodes AS mssmt_node + ON leaf.leaf_node_key = mssmt_node.key + AND leaf.leaf_node_namespace = mssmt_node.namespace -- Join on genesis_info_view to get leaf related fields. -JOIN genesis_info_view genesis +JOIN genesis_info_view AS genesis ON leaf.asset_genesis_id = genesis.gen_asset_id - -JOIN universe_servers as server +JOIN universe_servers AS server ON server.id = log.servers_id - -JOIN universe_roots as root +JOIN universe_roots AS root ON root.id = log.universe_root_id - -WHERE (log.sync_direction = sqlc.narg('sync_direction') - OR sqlc.narg('sync_direction') IS NULL) - AND - (log.status = sqlc.narg('status') OR sqlc.narg('status') IS NULL) - AND - +WHERE (log.sync_direction = sqlc.narg('sync_direction') OR sqlc.narg('sync_direction') IS NULL) + AND (log.status = sqlc.narg('status') OR sqlc.narg('status') IS NULL) -- Universe leaves WHERE clauses. - (leaf.leaf_node_namespace = sqlc.narg('leaf_namespace') - OR sqlc.narg('leaf_namespace') IS NULL) - AND - (leaf.minting_point = sqlc.narg('leaf_minting_point_bytes') - OR sqlc.narg('leaf_minting_point_bytes') IS NULL) - AND - (leaf.script_key_bytes = sqlc.narg('leaf_script_key_bytes') - OR sqlc.narg('leaf_script_key_bytes') IS NULL); + AND (leaf.leaf_node_namespace = sqlc.narg('leaf_namespace') OR sqlc.narg('leaf_namespace') IS NULL) + AND (leaf.minting_point = sqlc.narg('leaf_minting_point_bytes') OR sqlc.narg('leaf_minting_point_bytes') IS NULL) + AND (leaf.script_key_bytes = sqlc.narg('leaf_script_key_bytes') OR sqlc.narg('leaf_script_key_bytes') IS NULL); -- name: DeleteFederationProofSyncLog :exec WITH selected_server_id AS ( diff --git a/tapdb/sqlc/universe.sql.go b/tapdb/sqlc/universe.sql.go index 9049862b7..da202d177 100644 --- a/tapdb/sqlc/universe.sql.go +++ b/tapdb/sqlc/universe.sql.go @@ -71,7 +71,7 @@ WITH root_id AS ( WHERE namespace_root = $1 ) DELETE FROM universe_events -WHERE universe_root_id = (SELECT id from root_id) +WHERE universe_root_id = (SELECT id FROM root_id) ` func (q *Queries) DeleteUniverseEvents(ctx context.Context, namespaceRoot string) error { @@ -140,7 +140,7 @@ func (q *Queries) FetchMultiverseRoot(ctx context.Context, namespaceRoot string) const fetchUniverseKeys = `-- name: FetchUniverseKeys :many SELECT leaves.minting_point, leaves.script_key_bytes -FROM universe_leaves leaves +FROM universe_leaves AS leaves WHERE leaves.leaf_node_namespace = $1 ORDER BY CASE WHEN $2 = 0 THEN leaves.id END ASC, @@ -196,10 +196,10 @@ FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets - ON genesis_assets.asset_id = universe_roots.asset_id + ON genesis_assets.asset_id = universe_roots.asset_id WHERE mssmt_nodes.namespace = $1 ` @@ -364,8 +364,8 @@ SELECT SUM(CASE WHEN event_type = 'SYNC' THEN 1 ELSE 0 END) AS sync_events, SUM(CASE WHEN event_type = 'NEW_PROOF' THEN 1 ELSE 0 END) AS new_proof_events FROM universe_events -WHERE event_type IN ('SYNC', 'NEW_PROOF') AND - event_timestamp >= $1 AND event_timestamp <= $2 +WHERE event_type IN ('SYNC', 'NEW_PROOF') + AND event_timestamp BETWEEN $1 AND $2 GROUP BY day ORDER BY day ` @@ -482,56 +482,33 @@ func (q *Queries) QueryFederationGlobalSyncConfigs(ctx context.Context) ([]Feder const queryFederationProofSyncLog = `-- name: QueryFederationProofSyncLog :many SELECT log.id, status, timestamp, sync_direction, attempt_counter, - - -- Select fields from the universe_servers table. - server.id as server_id, + server.id AS server_id, server.server_host, - - -- Select universe leaf related fields. - leaf.minting_point as leaf_minting_point_bytes, - leaf.script_key_bytes as leaf_script_key_bytes, - mssmt_node.value as leaf_genesis_proof, - genesis.gen_asset_id as leaf_gen_asset_id, - genesis.asset_id as leaf_asset_id, - - -- Select fields from the universe_roots table. - root.asset_id as uni_asset_id, - root.group_key as uni_group_key, - root.proof_type as uni_proof_type - -FROM federation_proof_sync_log as log - -JOIN universe_leaves as leaf + leaf.minting_point AS leaf_minting_point_bytes, + leaf.script_key_bytes AS leaf_script_key_bytes, + mssmt_node.value AS leaf_genesis_proof, + genesis.gen_asset_id AS leaf_gen_asset_id, + genesis.asset_id AS leaf_asset_id, + root.asset_id AS uni_asset_id, + root.group_key AS uni_group_key, + root.proof_type AS uni_proof_type +FROM federation_proof_sync_log AS log +JOIN universe_leaves AS leaf ON leaf.id = log.proof_leaf_id - -JOIN mssmt_nodes mssmt_node - ON leaf.leaf_node_key = mssmt_node.key AND - leaf.leaf_node_namespace = mssmt_node.namespace - -JOIN genesis_info_view genesis +JOIN mssmt_nodes AS mssmt_node + ON leaf.leaf_node_key = mssmt_node.key + AND leaf.leaf_node_namespace = mssmt_node.namespace +JOIN genesis_info_view AS genesis ON leaf.asset_genesis_id = genesis.gen_asset_id - -JOIN universe_servers as server +JOIN universe_servers AS server ON server.id = log.servers_id - -JOIN universe_roots as root +JOIN universe_roots AS root ON root.id = log.universe_root_id - -WHERE (log.sync_direction = $1 - OR $1 IS NULL) - AND - (log.status = $2 OR $2 IS NULL) - AND - - -- Universe leaves WHERE clauses. - (leaf.leaf_node_namespace = $3 - OR $3 IS NULL) - AND - (leaf.minting_point = $4 - OR $4 IS NULL) - AND - (leaf.script_key_bytes = $5 - OR $5 IS NULL) +WHERE (log.sync_direction = $1 OR $1 IS NULL) + AND (log.status = $2 OR $2 IS NULL) + AND (leaf.leaf_node_namespace = $3 OR $3 IS NULL) + AND (leaf.minting_point = $4 OR $4 IS NULL) + AND (leaf.script_key_bytes = $5 OR $5 IS NULL) ` type QueryFederationProofSyncLogParams struct { @@ -560,8 +537,6 @@ type QueryFederationProofSyncLogRow struct { UniProofType string } -// Join on mssmt_nodes to get leaf related fields. -// Join on genesis_info_view to get leaf related fields. func (q *Queries) QueryFederationProofSyncLog(ctx context.Context, arg QueryFederationProofSyncLogParams) ([]QueryFederationProofSyncLogRow, error) { rows, err := q.db.QueryContext(ctx, queryFederationProofSyncLog, arg.SyncDirection, @@ -865,21 +840,19 @@ func (q *Queries) QueryUniverseAssetStats(ctx context.Context, arg QueryUniverse } const queryUniverseLeaves = `-- name: QueryUniverseLeaves :many -SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value genesis_proof, - nodes.sum sum_amt, gen.asset_id -FROM universe_leaves leaves -JOIN mssmt_nodes nodes - ON leaves.leaf_node_key = nodes.key AND - leaves.leaf_node_namespace = nodes.namespace -JOIN genesis_info_view gen +SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value AS genesis_proof, + nodes.sum AS sum_amt, gen.asset_id +FROM universe_leaves AS leaves +JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace +JOIN genesis_info_view AS gen ON leaves.asset_genesis_id = gen.gen_asset_id WHERE leaves.leaf_node_namespace = $1 - AND - (leaves.minting_point = $2 OR - $2 IS NULL) - AND - (leaves.script_key_bytes = $3 OR - $3 IS NULL) + AND (leaves.minting_point = $2 OR + $2 IS NULL) + AND (leaves.script_key_bytes = $3 OR + $3 IS NULL) ` type QueryUniverseLeavesParams struct { @@ -1058,14 +1031,14 @@ func (q *Queries) UniverseLeaves(ctx context.Context) ([]UniverseLeafe, error) { const universeRoots = `-- name: UniverseRoots :many SELECT universe_roots.asset_id, group_key, proof_type, - mssmt_roots.root_hash root_hash, mssmt_nodes.sum root_sum, - genesis_assets.asset_tag asset_name + mssmt_roots.root_hash AS root_hash, mssmt_nodes.sum AS root_sum, + genesis_assets.asset_tag AS asset_name FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets ON genesis_assets.asset_id = universe_roots.asset_id ORDER BY @@ -1142,7 +1115,7 @@ func (q *Queries) UpsertFederationGlobalSyncConfig(ctx context.Context, arg Upse } const upsertFederationProofSyncLog = `-- name: UpsertFederationProofSyncLog :one -INSERT INTO federation_proof_sync_log as log ( +INSERT INTO federation_proof_sync_log AS log ( status, timestamp, sync_direction, proof_leaf_id, universe_root_id, servers_id ) VALUES ( @@ -1176,7 +1149,7 @@ DO UPDATE SET timestamp = EXCLUDED.timestamp, -- Increment the attempt counter. attempt_counter = CASE - WHEN $9 = true THEN log.attempt_counter + 1 + WHEN $9 = TRUE THEN log.attempt_counter + 1 ELSE log.attempt_counter END RETURNING id @@ -1342,8 +1315,6 @@ INSERT INTO universe_roots ( ) VALUES ( $1, $2, $3, $4 ) ON CONFLICT (namespace_root) - -- This is a NOP, namespace_root is the unique field that caused the - -- conflict. DO UPDATE SET namespace_root = EXCLUDED.namespace_root RETURNING id ` diff --git a/tapdb/universe_perf_test.go b/tapdb/universe_perf_test.go index 7bbcab0d0..3730c3790 100644 --- a/tapdb/universe_perf_test.go +++ b/tapdb/universe_perf_test.go @@ -2,11 +2,13 @@ package tapdb import ( "context" + "database/sql" "fmt" "sort" "testing" "time" + "github.com/lightninglabs/taproot-assets/tapdb/sqlc" "github.com/lightningnetwork/lnd/clock" "github.com/stretchr/testify/require" ) @@ -20,97 +22,80 @@ type dbSizeStats struct { totalSize int64 // dataSize + indexSize } -// measureDBSize gets table and index sizes from SQLite -func measureDBSize(t *testing.T, db *BaseDB) map[string]*dbSizeStats { +// measureDBSize fetches the size of tables and indexes for PostgreSQL and SQLite databases. +func measureDBSize(t *testing.T, db *BaseDB, isPostgres bool) map[string]*dbSizeStats { stats := make(map[string]*dbSizeStats) - - // First get list of all tables - rows, err := db.Query(` - SELECT DISTINCT - tbl_name, - type - FROM sqlite_master - WHERE type='table' - `) - require.NoError(t, err) - defer rows.Close() - - tables := make([]string, 0) - for rows.Next() { - var name, tblType string - err := rows.Scan(&name, &tblType) + var rows *sql.Rows + var err error + + if isPostgres { + // Fetch table list and their size details for PostgreSQL + rows, err = db.Query(` + SELECT tablename, pg_table_size(tablename::regclass) AS data_size, + pg_indexes_size(tablename::regclass) AS index_size + FROM pg_catalog.pg_tables + WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema' + `) require.NoError(t, err) - tables = append(tables, name) - - stats[name] = &dbSizeStats{ - tableName: name, - } - } - - // For each table, get its stats - for _, tableName := range tables { - // Get row count using COUNT(*) - var rowCount int64 - err = db.QueryRow(fmt.Sprintf(`SELECT COUNT(*) FROM main.%q`, tableName)).Scan(&rowCount) - if err != nil { - t.Logf("Skipping row count for %s: %v", tableName, err) - continue + defer rows.Close() + + for rows.Next() { + var tableName string + var dataSize, indexSize int64 + err := rows.Scan(&tableName, &dataSize, &indexSize) + require.NoError(t, err) + + stats[tableName] = &dbSizeStats{ + tableName: tableName, + dataSize: dataSize, + indexSize: indexSize, + totalSize: dataSize + indexSize, + } } - stats[tableName].rowCount = rowCount - - // Get table size - var pageCount int64 - err = db.QueryRow(` - SELECT COUNT(*) - FROM dbstat - WHERE name = ? - `, tableName).Scan(&pageCount) - if err != nil { - t.Logf("Skipping size stats for %s: %v", tableName, err) - continue + } else { + // SQLite: Fetch tables, calculate data size, and index size from dbstat + rows, err = db.Query(` + SELECT DISTINCT tbl_name + FROM sqlite_master + WHERE type='table' + `) + require.NoError(t, err) + defer rows.Close() + + tables := make([]string, 0) + for rows.Next() { + var name string + err := rows.Scan(&name) + require.NoError(t, err) + tables = append(tables, name) + stats[name] = &dbSizeStats{ + tableName: name, + } } - // Get page size (constant for the database) - var pageSize int64 - err = db.QueryRow(`PRAGMA page_size`).Scan(&pageSize) - require.NoError(t, err) + // Calculate sizes for each table using dbstat for SQLite + for _, tableName := range tables { + var pageCount, pageSize, indexSize int64 - stats[tableName].dataSize = pageCount * pageSize - } + err = db.QueryRow(`SELECT COUNT(*) FROM dbstat WHERE name = ?`, tableName).Scan(&pageCount) + if err != nil { + t.Logf("Skipping size stats for %s: %v", tableName, err) + continue + } + err = db.QueryRow(`PRAGMA page_size`).Scan(&pageSize) + require.NoError(t, err) - // Get list of indices and their sizes - rows, err = db.Query(` - SELECT - m.tbl_name as table_name, - m.name as index_name, - (SELECT COUNT(*) FROM dbstat WHERE name = m.name) as page_count, - (SELECT page_size FROM pragma_page_size) as page_size - FROM sqlite_master m - WHERE m.type = 'index' - `) - require.NoError(t, err) - defer rows.Close() - - for rows.Next() { - var ( - tableName string - indexName string - pageCount int64 - pageSize int64 - ) - err := rows.Scan(&tableName, &indexName, &pageCount, &pageSize) - if err != nil { - t.Logf("Skipping index stat: %v", err) - continue - } + err = db.QueryRow(`SELECT COALESCE(SUM(pgsize), 0) FROM dbstat WHERE name = ? AND pagetype = 'index'`, tableName).Scan(&indexSize) + if err != nil { + t.Logf("Skipping index size for %s: %v", tableName, err) + continue + } - if stat, ok := stats[tableName]; ok { - indexSize := pageCount * pageSize - stat.indexSize += indexSize - stat.totalSize = stat.dataSize + stat.indexSize + stats[tableName].dataSize = pageCount * pageSize + stats[tableName].indexSize = indexSize + stats[tableName].totalSize = stats[tableName].dataSize + stats[tableName].indexSize } } - return stats } @@ -191,10 +176,10 @@ func TestUniverseIndexPerformance(t *testing.T) { t.Parallel() const ( - numAssets = 25 + numAssets = 50 numLeavesPerTree = 10 - numEventsPerAsset = 15 - numQueries = 10 + numEventsPerAsset = 25 + numQueries = 50 batchSize = 5 ) @@ -210,7 +195,7 @@ func TestUniverseIndexPerformance(t *testing.T) { runTest := func(withIndices bool) { t.Run(fmt.Sprintf("indices=%v", withIndices), func(t *testing.T) { ctx, cancel := context.WithTimeout( - context.Background(), time.Minute, + context.Background(), time.Minute * 5, ) defer cancel() @@ -279,7 +264,8 @@ func TestUniverseIndexPerformance(t *testing.T) { // Measure size after data creation t.Log("Measuring initial database size...") - initialSizes := measureDBSize(t, db.BaseDB) + isPostgres := db.Backend() == sqlc.BackendTypePostgres + initialSizes := measureDBSize(t, db.BaseDB, isPostgres) prettyPrintSizeStats(t, initialSizes) if withIndices { @@ -318,15 +304,28 @@ func TestUniverseIndexPerformance(t *testing.T) { start := time.Now() err := stats.db.ExecTx(ctx, &readTx, func(db UniverseStatsStore) error { - _, err := db.QueryAssetStatsPerDaySqlite( - ctx, AssetStatsPerDayQuery{ + if isPostgres { + _, err := db.QueryAssetStatsPerDayPostgres( + ctx, AssetStatsPerDayQueryPg{ StartTime: testClock.Now().Add( -24 * time.Hour, ).Unix(), EndTime: testClock.Now().Unix(), }, - ) - return err + ) + require.NoError(t, err) + } else { + _, err := db.QueryAssetStatsPerDaySqlite( + ctx, AssetStatsPerDayQuery{ + StartTime: testClock.Now().Add( + -24 * time.Hour, + ).Unix(), + EndTime: testClock.Now().Unix(), + }, + ) + require.NoError(t, err) + } + return nil }) require.NoError(t, err) return time.Since(start) @@ -345,7 +344,8 @@ func TestUniverseIndexPerformance(t *testing.T) { Namespace: h.assetUniverses[0].id.String(), }, ) - return err + require.NoError(t, err) + return nil }) require.NoError(t, err) return time.Since(start)