From 5c0db9d1117762552255039dffabb5d29353014c Mon Sep 17 00:00:00 2001 From: Jason Brill Date: Fri, 8 Nov 2024 10:23:41 -0500 Subject: [PATCH 1/3] tapdb: universe indices for optimized querying --- tapdb/migrations.go | 2 +- ...erse_optimization_indexes_queries.down.sql | 5 + ...iverse_optimization_indexes_queries.up.sql | 14 +++ tapdb/sqlc/querier.go | 1 + tapdb/sqlc/queries/universe.sql | 114 ++++++++---------- tapdb/sqlc/universe.sql.go | 114 ++++++++---------- 6 files changed, 115 insertions(+), 135 deletions(-) create mode 100644 tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql create mode 100644 tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql diff --git a/tapdb/migrations.go b/tapdb/migrations.go index 7b3f1a6fb..e49bc6e1d 100644 --- a/tapdb/migrations.go +++ b/tapdb/migrations.go @@ -22,7 +22,7 @@ const ( // daemon. // // NOTE: This MUST be updated when a new migration is added. - LatestMigrationVersion = 23 + LatestMigrationVersion = 24 ) // MigrationTarget is a functional option that can be passed to applyMigrations diff --git a/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql new file mode 100644 index 000000000..eefb73d62 --- /dev/null +++ b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.down.sql @@ -0,0 +1,5 @@ +DROP INDEX IF EXISTS idx_mssmt_nodes_stats; + +DROP INDEX IF EXISTS idx_universe_leaves_composite; + +DROP INDEX IF EXISTS idx_universe_roots_composite; diff --git a/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql new file mode 100644 index 000000000..21f92e967 --- /dev/null +++ b/tapdb/sqlc/migrations/000024_universe_optimization_indexes_queries.up.sql @@ -0,0 +1,14 @@ +-- Most impactful for query_asset_stats which currently has highest latency +-- Supports the common join pattern and filters on proof_type. +CREATE INDEX IF NOT EXISTS idx_universe_leaves_asset +ON universe_leaves(asset_genesis_id, universe_root_id); + +-- Helps with the join conditions we frequently see +-- This is especially useful for query_universe_leaves and improves join efficiency. +CREATE INDEX IF NOT EXISTS idx_mssmt_nodes_composite +ON mssmt_nodes(namespace, key, hash_key, sum); + +-- Optimizes the common namespace_root lookups along with proof_type filtering +-- This helps with fetch_universe_root and roots-related queries. +CREATE INDEX IF NOT EXISTS idx_universe_roots_composite +ON universe_roots(namespace_root, proof_type, asset_id); \ No newline at end of file diff --git a/tapdb/sqlc/querier.go b/tapdb/sqlc/querier.go index 62c213c34..ccf31d99d 100644 --- a/tapdb/sqlc/querier.go +++ b/tapdb/sqlc/querier.go @@ -117,6 +117,7 @@ type Querier interface { // around that needs to be used with this query until a sqlc bug is fixed. QueryAssetBalancesByAsset(ctx context.Context, arg QueryAssetBalancesByAssetParams) ([]QueryAssetBalancesByAssetRow, error) QueryAssetBalancesByGroup(ctx context.Context, arg QueryAssetBalancesByGroupParams) ([]QueryAssetBalancesByGroupRow, error) + // BETWEEN is inclusive for both start and end values. QueryAssetStatsPerDayPostgres(ctx context.Context, arg QueryAssetStatsPerDayPostgresParams) ([]QueryAssetStatsPerDayPostgresRow, error) QueryAssetStatsPerDaySqlite(ctx context.Context, arg QueryAssetStatsPerDaySqliteParams) ([]QueryAssetStatsPerDaySqliteRow, error) QueryAssetTransfers(ctx context.Context, arg QueryAssetTransfersParams) ([]QueryAssetTransfersRow, error) diff --git a/tapdb/sqlc/queries/universe.sql b/tapdb/sqlc/queries/universe.sql index 787162906..ac7ddebaf 100644 --- a/tapdb/sqlc/queries/universe.sql +++ b/tapdb/sqlc/queries/universe.sql @@ -6,10 +6,10 @@ FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets - ON genesis_assets.asset_id = universe_roots.asset_id + ON genesis_assets.asset_id = universe_roots.asset_id WHERE mssmt_nodes.namespace = @namespace; -- name: UpsertUniverseRoot :one @@ -30,7 +30,7 @@ WITH root_id AS ( WHERE namespace_root = @namespace_root ) DELETE FROM universe_events -WHERE universe_root_id = (SELECT id from root_id); +WHERE universe_root_id = (SELECT id FROM root_id); -- name: DeleteUniverseRoot :exec DELETE FROM universe_roots @@ -54,25 +54,23 @@ DELETE FROM universe_leaves WHERE leaf_node_namespace = @namespace; -- name: QueryUniverseLeaves :many -SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value genesis_proof, - nodes.sum sum_amt, gen.asset_id -FROM universe_leaves leaves -JOIN mssmt_nodes nodes - ON leaves.leaf_node_key = nodes.key AND - leaves.leaf_node_namespace = nodes.namespace -JOIN genesis_info_view gen +SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value AS genesis_proof, + nodes.sum AS sum_amt, gen.asset_id +FROM universe_leaves AS leaves +JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace +JOIN genesis_info_view AS gen ON leaves.asset_genesis_id = gen.gen_asset_id WHERE leaves.leaf_node_namespace = @namespace - AND - (leaves.minting_point = sqlc.narg('minting_point_bytes') OR - sqlc.narg('minting_point_bytes') IS NULL) - AND - (leaves.script_key_bytes = sqlc.narg('script_key_bytes') OR - sqlc.narg('script_key_bytes') IS NULL); + AND (leaves.minting_point = sqlc.narg('minting_point_bytes') OR + sqlc.narg('minting_point_bytes') IS NULL) + AND (leaves.script_key_bytes = sqlc.narg('script_key_bytes') OR + sqlc.narg('script_key_bytes') IS NULL); -- name: FetchUniverseKeys :many SELECT leaves.minting_point, leaves.script_key_bytes -FROM universe_leaves leaves +FROM universe_leaves AS leaves WHERE leaves.leaf_node_namespace = @namespace ORDER BY CASE WHEN sqlc.narg('sort_direction') = 0 THEN leaves.id END ASC, @@ -84,14 +82,14 @@ SELECT * FROM universe_leaves; -- name: UniverseRoots :many SELECT universe_roots.asset_id, group_key, proof_type, - mssmt_roots.root_hash root_hash, mssmt_nodes.sum root_sum, - genesis_assets.asset_tag asset_name + mssmt_roots.root_hash AS root_hash, mssmt_nodes.sum AS root_sum, + genesis_assets.asset_tag AS asset_name FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets ON genesis_assets.asset_id = universe_roots.asset_id ORDER BY @@ -329,8 +327,9 @@ SELECT SUM(CASE WHEN event_type = 'SYNC' THEN 1 ELSE 0 END) AS sync_events, SUM(CASE WHEN event_type = 'NEW_PROOF' THEN 1 ELSE 0 END) AS new_proof_events FROM universe_events -WHERE event_type IN ('SYNC', 'NEW_PROOF') AND - event_timestamp >= @start_time AND event_timestamp <= @end_time +-- BETWEEN is inclusive for both start and end values. +WHERE event_type IN ('SYNC', 'NEW_PROOF') + AND event_timestamp BETWEEN @start_time AND @end_time GROUP BY day ORDER BY day; @@ -367,7 +366,7 @@ FROM federation_uni_sync_config ORDER BY group_key NULLS LAST, asset_id NULLS LAST, proof_type; -- name: UpsertFederationProofSyncLog :one -INSERT INTO federation_proof_sync_log as log ( +INSERT INTO federation_proof_sync_log AS log ( status, timestamp, sync_direction, proof_leaf_id, universe_root_id, servers_id ) VALUES ( @@ -401,7 +400,7 @@ DO UPDATE SET timestamp = EXCLUDED.timestamp, -- Increment the attempt counter. attempt_counter = CASE - WHEN @bump_sync_attempt_counter = true THEN log.attempt_counter + 1 + WHEN @bump_sync_attempt_counter = TRUE THEN log.attempt_counter + 1 ELSE log.attempt_counter END RETURNING id; @@ -409,58 +408,39 @@ RETURNING id; -- name: QueryFederationProofSyncLog :many SELECT log.id, status, timestamp, sync_direction, attempt_counter, - -- Select fields from the universe_servers table. - server.id as server_id, + server.id AS server_id, server.server_host, - -- Select universe leaf related fields. - leaf.minting_point as leaf_minting_point_bytes, - leaf.script_key_bytes as leaf_script_key_bytes, - mssmt_node.value as leaf_genesis_proof, - genesis.gen_asset_id as leaf_gen_asset_id, - genesis.asset_id as leaf_asset_id, - + leaf.minting_point AS leaf_minting_point_bytes, + leaf.script_key_bytes AS leaf_script_key_bytes, + mssmt_node.value AS leaf_genesis_proof, + genesis.gen_asset_id AS leaf_gen_asset_id, + genesis.asset_id AS leaf_asset_id, -- Select fields from the universe_roots table. - root.asset_id as uni_asset_id, - root.group_key as uni_group_key, - root.proof_type as uni_proof_type - -FROM federation_proof_sync_log as log - -JOIN universe_leaves as leaf + root.asset_id AS uni_asset_id, + root.group_key AS uni_group_key, + root.proof_type AS uni_proof_type +FROM federation_proof_sync_log AS log +JOIN universe_leaves AS leaf ON leaf.id = log.proof_leaf_id - -- Join on mssmt_nodes to get leaf related fields. -JOIN mssmt_nodes mssmt_node - ON leaf.leaf_node_key = mssmt_node.key AND - leaf.leaf_node_namespace = mssmt_node.namespace - +JOIN mssmt_nodes AS mssmt_node + ON leaf.leaf_node_key = mssmt_node.key + AND leaf.leaf_node_namespace = mssmt_node.namespace -- Join on genesis_info_view to get leaf related fields. -JOIN genesis_info_view genesis +JOIN genesis_info_view AS genesis ON leaf.asset_genesis_id = genesis.gen_asset_id - -JOIN universe_servers as server +JOIN universe_servers AS server ON server.id = log.servers_id - -JOIN universe_roots as root +JOIN universe_roots AS root ON root.id = log.universe_root_id - -WHERE (log.sync_direction = sqlc.narg('sync_direction') - OR sqlc.narg('sync_direction') IS NULL) - AND - (log.status = sqlc.narg('status') OR sqlc.narg('status') IS NULL) - AND - +WHERE (log.sync_direction = sqlc.narg('sync_direction') OR sqlc.narg('sync_direction') IS NULL) + AND (log.status = sqlc.narg('status') OR sqlc.narg('status') IS NULL) -- Universe leaves WHERE clauses. - (leaf.leaf_node_namespace = sqlc.narg('leaf_namespace') - OR sqlc.narg('leaf_namespace') IS NULL) - AND - (leaf.minting_point = sqlc.narg('leaf_minting_point_bytes') - OR sqlc.narg('leaf_minting_point_bytes') IS NULL) - AND - (leaf.script_key_bytes = sqlc.narg('leaf_script_key_bytes') - OR sqlc.narg('leaf_script_key_bytes') IS NULL); + AND (leaf.leaf_node_namespace = sqlc.narg('leaf_namespace') OR sqlc.narg('leaf_namespace') IS NULL) + AND (leaf.minting_point = sqlc.narg('leaf_minting_point_bytes') OR sqlc.narg('leaf_minting_point_bytes') IS NULL) + AND (leaf.script_key_bytes = sqlc.narg('leaf_script_key_bytes') OR sqlc.narg('leaf_script_key_bytes') IS NULL); -- name: DeleteFederationProofSyncLog :exec WITH selected_server_id AS ( diff --git a/tapdb/sqlc/universe.sql.go b/tapdb/sqlc/universe.sql.go index 9049862b7..fd6825628 100644 --- a/tapdb/sqlc/universe.sql.go +++ b/tapdb/sqlc/universe.sql.go @@ -71,7 +71,7 @@ WITH root_id AS ( WHERE namespace_root = $1 ) DELETE FROM universe_events -WHERE universe_root_id = (SELECT id from root_id) +WHERE universe_root_id = (SELECT id FROM root_id) ` func (q *Queries) DeleteUniverseEvents(ctx context.Context, namespaceRoot string) error { @@ -140,7 +140,7 @@ func (q *Queries) FetchMultiverseRoot(ctx context.Context, namespaceRoot string) const fetchUniverseKeys = `-- name: FetchUniverseKeys :many SELECT leaves.minting_point, leaves.script_key_bytes -FROM universe_leaves leaves +FROM universe_leaves AS leaves WHERE leaves.leaf_node_namespace = $1 ORDER BY CASE WHEN $2 = 0 THEN leaves.id END ASC, @@ -196,10 +196,10 @@ FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets - ON genesis_assets.asset_id = universe_roots.asset_id + ON genesis_assets.asset_id = universe_roots.asset_id WHERE mssmt_nodes.namespace = $1 ` @@ -364,8 +364,8 @@ SELECT SUM(CASE WHEN event_type = 'SYNC' THEN 1 ELSE 0 END) AS sync_events, SUM(CASE WHEN event_type = 'NEW_PROOF' THEN 1 ELSE 0 END) AS new_proof_events FROM universe_events -WHERE event_type IN ('SYNC', 'NEW_PROOF') AND - event_timestamp >= $1 AND event_timestamp <= $2 +WHERE event_type IN ('SYNC', 'NEW_PROOF') + AND event_timestamp BETWEEN $1 AND $2 GROUP BY day ORDER BY day ` @@ -381,6 +381,7 @@ type QueryAssetStatsPerDayPostgresRow struct { NewProofEvents int64 } +// BETWEEN is inclusive for both start and end values. func (q *Queries) QueryAssetStatsPerDayPostgres(ctx context.Context, arg QueryAssetStatsPerDayPostgresParams) ([]QueryAssetStatsPerDayPostgresRow, error) { rows, err := q.db.QueryContext(ctx, queryAssetStatsPerDayPostgres, arg.StartTime, arg.EndTime) if err != nil { @@ -482,56 +483,37 @@ func (q *Queries) QueryFederationGlobalSyncConfigs(ctx context.Context) ([]Feder const queryFederationProofSyncLog = `-- name: QueryFederationProofSyncLog :many SELECT log.id, status, timestamp, sync_direction, attempt_counter, - -- Select fields from the universe_servers table. - server.id as server_id, + server.id AS server_id, server.server_host, - -- Select universe leaf related fields. - leaf.minting_point as leaf_minting_point_bytes, - leaf.script_key_bytes as leaf_script_key_bytes, - mssmt_node.value as leaf_genesis_proof, - genesis.gen_asset_id as leaf_gen_asset_id, - genesis.asset_id as leaf_asset_id, - + leaf.minting_point AS leaf_minting_point_bytes, + leaf.script_key_bytes AS leaf_script_key_bytes, + mssmt_node.value AS leaf_genesis_proof, + genesis.gen_asset_id AS leaf_gen_asset_id, + genesis.asset_id AS leaf_asset_id, -- Select fields from the universe_roots table. - root.asset_id as uni_asset_id, - root.group_key as uni_group_key, - root.proof_type as uni_proof_type - -FROM federation_proof_sync_log as log - -JOIN universe_leaves as leaf + root.asset_id AS uni_asset_id, + root.group_key AS uni_group_key, + root.proof_type AS uni_proof_type +FROM federation_proof_sync_log AS log +JOIN universe_leaves AS leaf ON leaf.id = log.proof_leaf_id - -JOIN mssmt_nodes mssmt_node - ON leaf.leaf_node_key = mssmt_node.key AND - leaf.leaf_node_namespace = mssmt_node.namespace - -JOIN genesis_info_view genesis +JOIN mssmt_nodes AS mssmt_node + ON leaf.leaf_node_key = mssmt_node.key + AND leaf.leaf_node_namespace = mssmt_node.namespace +JOIN genesis_info_view AS genesis ON leaf.asset_genesis_id = genesis.gen_asset_id - -JOIN universe_servers as server +JOIN universe_servers AS server ON server.id = log.servers_id - -JOIN universe_roots as root +JOIN universe_roots AS root ON root.id = log.universe_root_id - -WHERE (log.sync_direction = $1 - OR $1 IS NULL) - AND - (log.status = $2 OR $2 IS NULL) - AND - +WHERE (log.sync_direction = $1 OR $1 IS NULL) + AND (log.status = $2 OR $2 IS NULL) -- Universe leaves WHERE clauses. - (leaf.leaf_node_namespace = $3 - OR $3 IS NULL) - AND - (leaf.minting_point = $4 - OR $4 IS NULL) - AND - (leaf.script_key_bytes = $5 - OR $5 IS NULL) + AND (leaf.leaf_node_namespace = $3 OR $3 IS NULL) + AND (leaf.minting_point = $4 OR $4 IS NULL) + AND (leaf.script_key_bytes = $5 OR $5 IS NULL) ` type QueryFederationProofSyncLogParams struct { @@ -865,21 +847,19 @@ func (q *Queries) QueryUniverseAssetStats(ctx context.Context, arg QueryUniverse } const queryUniverseLeaves = `-- name: QueryUniverseLeaves :many -SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value genesis_proof, - nodes.sum sum_amt, gen.asset_id -FROM universe_leaves leaves -JOIN mssmt_nodes nodes - ON leaves.leaf_node_key = nodes.key AND - leaves.leaf_node_namespace = nodes.namespace -JOIN genesis_info_view gen +SELECT leaves.script_key_bytes, gen.gen_asset_id, nodes.value AS genesis_proof, + nodes.sum AS sum_amt, gen.asset_id +FROM universe_leaves AS leaves +JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace +JOIN genesis_info_view AS gen ON leaves.asset_genesis_id = gen.gen_asset_id WHERE leaves.leaf_node_namespace = $1 - AND - (leaves.minting_point = $2 OR - $2 IS NULL) - AND - (leaves.script_key_bytes = $3 OR - $3 IS NULL) + AND (leaves.minting_point = $2 OR + $2 IS NULL) + AND (leaves.script_key_bytes = $3 OR + $3 IS NULL) ` type QueryUniverseLeavesParams struct { @@ -1058,14 +1038,14 @@ func (q *Queries) UniverseLeaves(ctx context.Context) ([]UniverseLeafe, error) { const universeRoots = `-- name: UniverseRoots :many SELECT universe_roots.asset_id, group_key, proof_type, - mssmt_roots.root_hash root_hash, mssmt_nodes.sum root_sum, - genesis_assets.asset_tag asset_name + mssmt_roots.root_hash AS root_hash, mssmt_nodes.sum AS root_sum, + genesis_assets.asset_tag AS asset_name FROM universe_roots JOIN mssmt_roots ON universe_roots.namespace_root = mssmt_roots.namespace JOIN mssmt_nodes - ON mssmt_nodes.hash_key = mssmt_roots.root_hash AND - mssmt_nodes.namespace = mssmt_roots.namespace + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace JOIN genesis_assets ON genesis_assets.asset_id = universe_roots.asset_id ORDER BY @@ -1142,7 +1122,7 @@ func (q *Queries) UpsertFederationGlobalSyncConfig(ctx context.Context, arg Upse } const upsertFederationProofSyncLog = `-- name: UpsertFederationProofSyncLog :one -INSERT INTO federation_proof_sync_log as log ( +INSERT INTO federation_proof_sync_log AS log ( status, timestamp, sync_direction, proof_leaf_id, universe_root_id, servers_id ) VALUES ( @@ -1176,7 +1156,7 @@ DO UPDATE SET timestamp = EXCLUDED.timestamp, -- Increment the attempt counter. attempt_counter = CASE - WHEN $9 = true THEN log.attempt_counter + 1 + WHEN $9 = TRUE THEN log.attempt_counter + 1 ELSE log.attempt_counter END RETURNING id From e12bbbca419d4337a5b170a3c53974f86aaab856 Mon Sep 17 00:00:00 2001 From: Jason Brill Date: Fri, 8 Nov 2024 10:57:15 -0500 Subject: [PATCH 2/3] tapdb: universe db query re-write and index performance tests --- make/testing_flags.mk | 5 + tapdb/universe_perf_long_test.go | 12 + tapdb/universe_perf_short_test.go | 9 + tapdb/universe_perf_test.go | 402 ++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+) create mode 100644 tapdb/universe_perf_long_test.go create mode 100644 tapdb/universe_perf_short_test.go create mode 100644 tapdb/universe_perf_test.go diff --git a/make/testing_flags.mk b/make/testing_flags.mk index f657d89e4..467bf86b5 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -54,6 +54,11 @@ ifeq ($(dbbackend),postgres) DEV_TAGS += test_db_postgres endif +# Run universe tests with increased scale for performance testing. +ifneq ($(long-tests),) +DEV_TAGS += longtests +endif + ifneq ($(tags),) DEV_TAGS += ${tags} endif diff --git a/tapdb/universe_perf_long_test.go b/tapdb/universe_perf_long_test.go new file mode 100644 index 000000000..3d2998cea --- /dev/null +++ b/tapdb/universe_perf_long_test.go @@ -0,0 +1,12 @@ +//go:build longtests + +package tapdb + +// longTestScale is the scale factor for long tests. +const longTestScale = 5 + +var ( + numAssets = 100 * longTestScale + numLeavesPerTree = 300 * longTestScale + numQueries = 100 * longTestScale +) diff --git a/tapdb/universe_perf_short_test.go b/tapdb/universe_perf_short_test.go new file mode 100644 index 000000000..95e35fa65 --- /dev/null +++ b/tapdb/universe_perf_short_test.go @@ -0,0 +1,9 @@ +//go:build !longtests + +package tapdb + +var ( + numAssets = 100 + numLeavesPerTree = 300 + numQueries = 100 +) diff --git a/tapdb/universe_perf_test.go b/tapdb/universe_perf_test.go new file mode 100644 index 000000000..35ac992c8 --- /dev/null +++ b/tapdb/universe_perf_test.go @@ -0,0 +1,402 @@ +package tapdb + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/lightninglabs/taproot-assets/tapdb/sqlc" + "github.com/lightningnetwork/lnd/clock" + "github.com/stretchr/testify/require" +) + +// Common indices for universe tables. +const ( + // createUniverseRootIndex optimizes lookups on universe roots. + createUniverseRootIndex = ` + CREATE INDEX IF NOT EXISTS idx_universe_roots_composite + ON universe_roots( + namespace_root, + proof_type, + asset_id + );` + + // createUniverseLeavesIndex optimizes universe leaf queries. + createUniverseLeavesIndex = ` + CREATE INDEX IF NOT EXISTS idx_universe_leaves_composite + ON universe_leaves( + leaf_node_namespace, + universe_root_id, + leaf_node_key + );` + + // createMSMTNodesIndex optimizes MSSMT node lookups. + createMSMTNodesIndex = ` + CREATE INDEX IF NOT EXISTS idx_mssmt_nodes_composite + ON mssmt_nodes( + namespace, + key, + hash_key, + sum + );` +) + +// executeSQLStatements executes a list of SQL statements with logging. +func executeSQLStatements(t *testing.T, db *BaseDB, statements []string, + action string) { + + t.Logf("%s indices...", action) + + for _, stmt := range statements { + _, err := db.Exec(stmt) + if err != nil { + t.Fatalf("Failed to %s index: %v\nStatement: %s", + action, err, stmt) + } + } +} + +// createIndices creates all required indices on the database. +func createIndices(t *testing.T, db *BaseDB) { + statements := []string{ + createUniverseRootIndex, + createUniverseLeavesIndex, + createMSMTNodesIndex, + } + + executeSQLStatements(t, db, statements, "Creating") +} + +// dropIndices removes all custom indices from the database. +func dropIndices(t *testing.T, db *BaseDB) { + statements := []string{ + `DROP INDEX IF EXISTS idx_universe_roots_composite`, + `DROP INDEX IF EXISTS idx_universe_leaves_composite`, + `DROP INDEX IF EXISTS idx_mssmt_nodes_composite`, + } + + executeSQLStatements(t, db, statements, "Dropping") +} + +// Query definitions for performance testing. +const ( + // fetchUniverseRootQuery returns root info for a given namespace. + fetchUniverseRootQuery = ` + SELECT + universe_roots.asset_id, + group_key, + proof_type, + mssmt_nodes.hash_key root_hash, + mssmt_nodes.sum root_sum, + genesis_assets.asset_tag asset_name + FROM universe_roots + JOIN mssmt_roots + ON universe_roots.namespace_root = mssmt_roots.namespace + JOIN mssmt_nodes + ON mssmt_nodes.hash_key = mssmt_roots.root_hash + AND mssmt_nodes.namespace = mssmt_roots.namespace + JOIN genesis_assets + ON genesis_assets.asset_id = universe_roots.asset_id + WHERE mssmt_nodes.namespace = $1` + + // queryUniverseLeavesQuery gets leaf info for a namespace. + queryUniverseLeavesQuery = ` + SELECT + leaves.script_key_bytes, + gen.gen_asset_id, + nodes.value AS genesis_proof, + nodes.sum AS sum_amt, + gen.asset_id + FROM universe_leaves AS leaves + JOIN mssmt_nodes AS nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace + JOIN genesis_info_view AS gen + ON leaves.asset_genesis_id = gen.gen_asset_id + WHERE leaves.leaf_node_namespace = $1 + AND (leaves.minting_point = $2 OR $2 IS NULL) + AND (leaves.script_key_bytes = $3 OR $3 IS NULL)` + + // queryAssetStatsQuery gets aggregated stats for assets. + queryAssetStatsQuery = ` + WITH asset_supply AS ( + SELECT + gen.asset_id, + SUM(nodes.sum) AS supply + FROM universe_leaves leaves + JOIN universe_roots roots + ON leaves.universe_root_id = roots.id + JOIN mssmt_nodes nodes + ON leaves.leaf_node_key = nodes.key + AND leaves.leaf_node_namespace = nodes.namespace + JOIN genesis_info_view gen + ON leaves.asset_genesis_id = gen.gen_asset_id + WHERE roots.proof_type = 'issuance' + GROUP BY gen.asset_id + ) + SELECT asset_id, supply, COUNT(*) as num_leaves + FROM asset_supply + GROUP BY asset_id, supply` +) + +// queryTest represents a test case for performance testing. +type queryTest struct { + name string + query string + args func(h *uniStatsHarness) []interface{} + dbTypes []sqlc.BackendType +} + +// testQueries defines the test cases and their compatible database types. +var testQueries = []queryTest{ + { + name: "fetch_universe_root", + query: fetchUniverseRootQuery, + args: func(h *uniStatsHarness) []interface{} { + return []interface{}{h.assetUniverses[0].id.String()} + }, + dbTypes: []sqlc.BackendType{ + sqlc.BackendTypeSqlite, + sqlc.BackendTypePostgres, + }, + }, + + { + name: "query_universe_leaves", + query: queryUniverseLeavesQuery, + args: func(h *uniStatsHarness) []interface{} { + return []interface{}{ + h.assetUniverses[0].id.String(), + nil, + nil, + } + }, + dbTypes: []sqlc.BackendType{ + sqlc.BackendTypeSqlite, + sqlc.BackendTypePostgres, + }, + }, + + { + name: "query_asset_stats", + query: queryAssetStatsQuery, + args: func(h *uniStatsHarness) []interface{} { + return []interface{}{} + }, + dbTypes: []sqlc.BackendType{ + sqlc.BackendTypeSqlite, + sqlc.BackendTypePostgres, + }, + }, +} + +// logPerformanceAnalysis prints performance metrics for all test queries. +func logPerformanceAnalysis(t *testing.T, results map[string]*queryStats) { + t.Log("\n=== Performance Analysis ===") + + for name, result := range results { + t.Logf("\nQuery: %s (%d runs)", name, result.queries) + + t.Log("Query Plan:") + t.Log(result.queryPlan) + + t.Logf("No indices exec time: %v", result.withoutIndices) + t.Logf("With indices exec time: %v", result.withIndices) + + // Calculate improvement factor. + var improvement float64 + if result.withIndices > 0 { + improvement = float64(result.withoutIndices) / + float64(result.withIndices) + } + + t.Logf("Improvement: %.2fx", improvement) + } +} + +// getQueryPlan retrieves the execution plan for a query from the database. +func getQueryPlan(ctx context.Context, t *testing.T, db *BaseDB, + q queryTest, h *uniStatsHarness) string { + + var ( + explainQuery string + plan strings.Builder + ) + + switch db.Backend() { + case sqlc.BackendTypeSqlite: + explainQuery = fmt.Sprintf("EXPLAIN QUERY PLAN %s", q.query) + rows, err := db.QueryContext(ctx, explainQuery, q.args(h)...) + require.NoError(t, err) + + for rows.Next() { + var ( + selectid, order, from int + detail string + ) + err := rows.Scan(&selectid, &order, &from, &detail) + require.NoError(t, err) + + plan.WriteString(fmt.Sprintf( + "--\nid: %d order: %d from: %d\n%s\n", + selectid, order, from, detail)) + } + rows.Close() + + case sqlc.BackendTypePostgres: + explainQuery = fmt.Sprintf("EXPLAIN %s", q.query) + rows, err := db.QueryContext(ctx, explainQuery, q.args(h)...) + require.NoError(t, err) + + for rows.Next() { + var planLine string + err := rows.Scan(&planLine) + if err != nil { + t.Fatalf("failed to scan plan line: %v", err) + } + plan.WriteString(planLine + "\n") + } + rows.Close() + + case sqlc.BackendTypeUnknown: + return "Unknown backend type" + } + + return plan.String() +} + +// executeQuery runs a query multiple times and measures execution time. +func executeQuery(ctx context.Context, t *testing.T, db *BaseDB, + q queryTest, h *uniStatsHarness) time.Duration { + + start := time.Now() + + for i := 0; i < numQueries; i++ { + rows, err := db.QueryContext(ctx, q.query, q.args(h)...) + require.NoError(t, err) + rows.Close() + } + + return time.Since(start) +} + +// supportsQuery checks if a query is supported by the given database type. +func supportsQuery(dbType sqlc.BackendType, + dbTypes []sqlc.BackendType) bool { + + for _, t := range dbTypes { + if t == dbType { + return true + } + } + + return false +} + +// queryStats tracks performance metrics for a single query test. +type queryStats struct { + name string + withoutIndices time.Duration + withIndices time.Duration + queries int + queryPlan string +} + +// TestUniverseIndexPerformance tests the performance impact +// of database indices. +func TestUniverseIndexPerformance(t *testing.T) { + t.Parallel() + + testResults := make(map[string]*queryStats) + + // runTest executes all queries with or without indices. + runTest := func(withIndices bool) { + testName := fmt.Sprintf("indices=%v", withIndices) + + t.Run(testName, func(t *testing.T) { + // Create context with reasonable timeout. + ctx, cancel := context.WithTimeout( + context.Background(), 5*time.Minute, + ) + defer cancel() + + db := NewTestDB(t) + sqlDB := db.BaseDB + + // Set reasonable connection limits. + sqlDB.SetMaxOpenConns(25) + sqlDB.SetMaxIdleConns(25) + sqlDB.SetConnMaxLifetime(time.Minute * 5) + + testClock := clock.NewTestClock(time.Now()) + statsDB, _ := newUniverseStatsWithDB( + db.BaseDB, + testClock, + ) + + // Add progress tracking. + t.Logf("Gen test data: %d assets, %d leaves/tree", + numAssets, numLeavesPerTree) + + h := newUniStatsHarness( + t, + numAssets, + db.BaseDB, + statsDB, + ) + require.NotNil(t, h) + + if withIndices { + createIndices(t, sqlDB) + } else { + dropIndices(t, sqlDB) + } + + // Execute each test query. + for _, q := range testQueries { + t.Logf("\n=== Query Plan for %s ===", q.name) + + // Skip unsupported queries. + if !supportsQuery(sqlDB.Backend(), q.dbTypes) { + t.Skipf("Query %s unsupported "+ + "by backend %v", + q.name, sqlDB.Backend()) + continue + } + + plan := getQueryPlan(ctx, t, sqlDB, q, h) + t.Logf("Query Plan:\n%s", plan) + + // Execute the query repeatedly. + queryTime := executeQuery(ctx, t, sqlDB, q, h) + + // Record results. + stat, ok := testResults[q.name] + if !ok { + stat = &queryStats{ + name: q.name, + } + testResults[q.name] = stat + } + + stat.queries = numQueries + stat.queryPlan = plan + if withIndices { + stat.withIndices = queryTime + } else { + stat.withoutIndices = queryTime + } + + t.Logf("%s executed in: %v", q.name, queryTime) + } + + logPerformanceAnalysis(t, testResults) + }) + } + + // Run tests with and without indices. + runTest(false) + runTest(true) +} From de6da1f066cb0912865c102ca244741e21eb9187 Mon Sep 17 00:00:00 2001 From: Jason Brill Date: Fri, 8 Nov 2024 10:57:40 -0500 Subject: [PATCH 3/3] tapdb: universe lock caching optimization --- tapdb/universe_stats.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tapdb/universe_stats.go b/tapdb/universe_stats.go index 709355931..fe110fa55 100644 --- a/tapdb/universe_stats.go +++ b/tapdb/universe_stats.go @@ -290,11 +290,11 @@ type UniverseStats struct { statsCacheLogger *cacheLogger statsRefresh *time.Timer - eventsMtx sync.Mutex + eventsMtx sync.RWMutex assetEventsCache assetEventsCache eventsCacheLogger *cacheLogger - syncStatsMtx sync.Mutex + syncStatsMtx sync.RWMutex syncStatsCache *atomicSyncStatsCache syncStatsRefresh *time.Timer } @@ -635,7 +635,9 @@ func (u *UniverseStats) QueryAssetStatsPerDay(ctx context.Context, // First, we'll check to see if we already have a cached result for // this query. query := newEventQuery(q) + u.eventsMtx.RLock() cachedResult, err := u.assetEventsCache.Get(query) + u.eventsMtx.RUnlock() if err == nil { u.eventsCacheLogger.Hit() return cachedResult, nil @@ -750,13 +752,16 @@ func (u *UniverseStats) QuerySyncStats(ctx context.Context, // First, check the cache to see if we already have a cached result for // this query. + u.syncStatsMtx.RLock() syncSnapshots := u.syncStatsCache.fetchQuery(q) + u.syncStatsMtx.RUnlock() + if syncSnapshots != nil { resp.SyncStats = syncSnapshots return resp, nil } - // Otherwise, we'll grab the main mutex so we can qury the db then + // Otherwise, we'll grab the main mutex so we can query the db then // cache the result. u.syncStatsMtx.Lock() defer u.syncStatsMtx.Unlock()