Switch to batched mode for validator list update. Update dependencies…

…. Bump version.
helikon-labs · Nov 26, 2024 · a1018e1 · a1018e1
1 parent 16c6a89
commit a1018e1
Show file tree

Hide file tree

Showing 42 changed files with 191 additions and 170 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/_config/base.toml b/_config/base.toml
@@ -67,6 +67,7 @@ start_block_number = 6_015_486
 
 [validator_list_updater]
 history_record_depth = 10
+db_fetch_batch_size = 200
 
 [dn]
 # this many most recent records will always be kept in the database for reference

diff --git a/_docker/compose/.env.sample b/_docker/compose/.env.sample
@@ -1,4 +1,4 @@
-VERSION=0.21.3
+VERSION=0.21.4
 ENV=production
 LOG_LEVEL=debug
 
@@ -67,7 +67,9 @@ POLKADOT_BLOCK_PROCESSOR_START_NUMBER=2700565
 
 # VALIDATOR LIST UPDATER
 KUSAMA_VALIDATOR_LIST_UPDATER_HISTORY_DEPTH=7
+KUSAMA_VALIDATOR_LIST_UPDATER_DB_FETCH_BATCH_SIZE=200
 POLKADOT_VALIDATOR_LIST_UPDATER_HISTORY_DEPTH=7
+POLKADOT_VALIDATOR_LIST_UPDATER_DB_FETCH_BATCH_SIZE=200
 
 # MONITORING
 PROMETHEUS_VERSION=v2.36.0

diff --git a/_docker/compose/05-docker-compose-subvt-kusama-services.yml b/_docker/compose/05-docker-compose-subvt-kusama-services.yml
@@ -45,6 +45,7 @@ services:
       - SUBVT__SUBSTRATE__RPC_URL=${KUSAMA_RPC_URL}
       # history depth
       - SUBVT__VALIDATOR_LIST_UPDATER__HISTORY_RECORD_DEPTH=${KUSAMA_VALIDATOR_LIST_UPDATER_HISTORY_DEPTH}
+      - SUBVT__VALIDATOR_LIST_UPDATER__DB_FETCH_BATCH_SIZE=${KUSAMA_VALIDATOR_LIST_UPDATER_DB_FETCH_BATCH_SIZE}
   subvt_kusama_active_validator_list_server:
     container_name: subvt_kusama_active_validator_list_server
     restart: unless-stopped

diff --git a/_docker/compose/06-docker-compose-subvt-polkadot-services.yml b/_docker/compose/06-docker-compose-subvt-polkadot-services.yml
@@ -45,6 +45,7 @@ services:
       - SUBVT__SUBSTRATE__RPC_URL=${POLKADOT_RPC_URL}
       # history depth
       - SUBVT__VALIDATOR_LIST_UPDATER__HISTORY_RECORD_DEPTH=${POLKADOT_VALIDATOR_LIST_UPDATER_HISTORY_DEPTH}
+      - SUBVT__VALIDATOR_LIST_UPDATER__DB_FETCH_BATCH_SIZE=${POLKADOT_VALIDATOR_LIST_UPDATER_DB_FETCH_BATCH_SIZE}
   subvt_polkadot_active_validator_list_server:
     container_name: subvt_polkadot_active_validator_list_server
     restart: unless-stopped

diff --git a/...ork/migrations/20241125143651_session_para_validator_validator_account_id_id_idx.down.sql b/...ork/migrations/20241125143651_session_para_validator_validator_account_id_id_idx.down.sql
@@ -0,0 +1 @@
+-- no-op
diff --git a/...twork/migrations/20241125143651_session_para_validator_validator_account_id_id_idx.up.sql b/...twork/migrations/20241125143651_session_para_validator_validator_account_id_id_idx.up.sql
@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS sub_session_para_validator_idx_validator_account_id_id
+    ON sub_session_para_validator (validator_account_id, id DESC);
diff --git a/_migrations/network/migrations/20241125150658_fx_validator_info_batch.down.sql b/_migrations/network/migrations/20241125150658_fx_validator_info_batch.down.sql
@@ -0,0 +1 @@
+DROP FUNCTION sub_get_validator_info_batch;
diff --git a/_migrations/network/migrations/20241125150658_fx_validator_info_batch.up.sql b/_migrations/network/migrations/20241125150658_fx_validator_info_batch.up.sql
@@ -0,0 +1,19 @@
+CREATE OR REPLACE FUNCTION sub_get_validator_info_batch (block_hash_param VARCHAR(66), account_ids_param VARCHAR(66)[], is_active_param boolean[], era_index_param bigint)
+RETURNS SETOF sub_validator_info
+AS $$
+
+DECLARE
+    validator_info sub_validator_info;
+    account_id VARCHAR(66);
+    i INT = 1;
+    is_active BOOLEAN = false;
+BEGIN
+    FOREACH account_id IN ARRAY account_ids_param
+    LOOP
+        is_active := is_active_param[i];
+        validator_info := sub_get_validator_info(block_hash_param, account_id, is_active, era_index_param);
+        RETURN NEXT validator_info;
+        i:= i + 1;
+    END LOOP;
+END
+$$ LANGUAGE plpgsql PARALLEL SAFE STABLE;
diff --git a/subvt-app-service/Cargo.toml b/subvt-app-service/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-app-service"
-version = "0.21.3"
+version = "0.21.4"
 edition = "2021"
 rust-version = "1.67.0"
 

diff --git a/subvt-block-processor/Cargo.toml b/subvt-block-processor/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "subvt-block-processor"
-version = "0.21.3"
+version = "0.21.4"
 edition = "2021"
 rust-version = "1.67.0"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		CREATE INDEX IF NOT EXISTS sub_session_para_validator_idx_validator_account_id_id
		ON sub_session_para_validator (validator_account_id, id DESC);