diff --git a/Cargo.lock b/Cargo.lock index 9581195c..eb53e8ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7696,7 +7696,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "subvt-app-service" -version = "0.21.4" +version = "0.22.0" dependencies = [ "actix-http", "actix-rt", @@ -7725,7 +7725,7 @@ dependencies = [ [[package]] name = "subvt-block-processor" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-lock", @@ -7751,7 +7751,7 @@ dependencies = [ [[package]] name = "subvt-config" -version = "0.21.4" +version = "0.22.0" dependencies = [ "config", "serde", @@ -7759,7 +7759,7 @@ dependencies = [ [[package]] name = "subvt-governance" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "lazy_static", @@ -7773,7 +7773,7 @@ dependencies = [ [[package]] name = "subvt-kline-updater" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -7796,7 +7796,7 @@ dependencies = [ [[package]] name = "subvt-logging" -version = "0.21.4" +version = "0.22.0" dependencies = [ "env_logger", "log", @@ -7805,7 +7805,7 @@ dependencies = [ [[package]] name = "subvt-metrics" -version = "0.21.4" +version = "0.22.0" dependencies = [ "env_logger", "log", @@ -7819,7 +7819,7 @@ dependencies = [ [[package]] name = "subvt-network-status-server" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -7844,7 +7844,7 @@ dependencies = [ [[package]] name = "subvt-network-status-updater" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -7866,7 +7866,7 @@ dependencies = [ [[package]] name = "subvt-nft" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "lazy_static", @@ -7878,7 +7878,7 @@ dependencies = [ [[package]] name = "subvt-notification-generator" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-lock", @@ -7905,7 +7905,7 @@ dependencies = [ [[package]] name = "subvt-notification-processor" -version = "0.21.4" +version = "0.22.0" dependencies = [ "a2", "anyhow", @@ -7940,7 +7940,7 @@ dependencies = [ [[package]] name = "subvt-onekv-updater" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -7962,7 +7962,7 @@ dependencies = [ [[package]] name = "subvt-persistence" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "chrono", @@ -7981,7 +7981,7 @@ dependencies = [ [[package]] name = "subvt-plotter" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "chrono", @@ -8004,7 +8004,7 @@ dependencies = [ [[package]] name = "subvt-proc-macro" -version = "0.21.4" +version = "0.22.0" dependencies = [ "proc-macro2", "quote", @@ -8013,7 +8013,7 @@ dependencies = [ [[package]] name = "subvt-referendum-updater" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -8037,7 +8037,7 @@ dependencies = [ [[package]] name = "subvt-report-service" -version = "0.21.4" +version = "0.22.0" dependencies = [ "actix-web", "anyhow", @@ -8065,7 +8065,7 @@ dependencies = [ [[package]] name = "subvt-service-common" -version = "0.21.4" +version = "0.22.0" dependencies = [ "actix-web", "anyhow", @@ -8080,9 +8080,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "subvt-session-validator-performance-updater" +version = "0.22.0" +dependencies = [ + "anyhow", + "async-lock", + "async-trait", + "lazy_static", + "log", + "once_cell", + "subvt-config", + "subvt-logging", + "subvt-metrics", + "subvt-persistence", + "subvt-service-common", + "subvt-types", + "tokio", +] + [[package]] name = "subvt-substrate-client" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-lock", @@ -8111,7 +8130,7 @@ dependencies = [ [[package]] name = "subvt-telegram-bot" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-recursion", @@ -8149,7 +8168,7 @@ dependencies = [ [[package]] name = "subvt-telemetry-processor" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-lock", @@ -8173,7 +8192,7 @@ dependencies = [ [[package]] name = "subvt-types" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "chrono", @@ -8215,7 +8234,7 @@ dependencies = [ [[package]] name = "subvt-utility" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "hex", @@ -8225,7 +8244,7 @@ dependencies = [ [[package]] name = "subvt-validator-details-server" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -8249,7 +8268,7 @@ dependencies = [ [[package]] name = "subvt-validator-list-server" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-trait", @@ -8276,7 +8295,7 @@ dependencies = [ [[package]] name = "subvt-validator-list-updater" -version = "0.21.4" +version = "0.22.0" dependencies = [ "anyhow", "async-lock", diff --git a/Cargo.toml b/Cargo.toml index 0899ff4a..0aaf64cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "subvt-referendum-updater", "subvt-report-service", "subvt-service-common", + "subvt-session-validator-performance-updater", "subvt-substrate-client", "subvt-telegram-bot", "subvt-telemetry-processor", diff --git a/_config/base.toml b/_config/base.toml index 87fc9511..d9610e28 100644 --- a/_config/base.toml +++ b/_config/base.toml @@ -145,6 +145,7 @@ validator_details_server_port = 11002 validator_list_updater_port = 11001 referendum_updater_port = 11014 kline_updater_port = 11015 +session_validator_performance_updater_port = 11016 [sub_id] api_url = "https://sub.id/api/v1/" @@ -160,4 +161,8 @@ tmp_dir_path = "/path/to/the/temporary/image/dir" sleep_seconds = 3600 begin_year = 2022 begin_month = 1 -begin_day = 1 \ No newline at end of file +begin_day = 1 + +[session_validator_performance_updater] +start_session_index = 5000 +sleep_seconds = 60 \ No newline at end of file diff --git a/_config/network/kusama.toml b/_config/network/kusama.toml index 43da5d58..11846b2c 100644 --- a/_config/network/kusama.toml +++ b/_config/network/kusama.toml @@ -18,4 +18,7 @@ start_block_number = 6_015_486 data_endpoint = "https://nodes.web3.foundation/api/cohort/1/kusama" [notification_generator] -unclaimed_payout_check_delay_hours = 3 \ No newline at end of file +unclaimed_payout_check_delay_hours = 3 + +[session_validator_performance_updater] +start_session_index = 5000 \ No newline at end of file diff --git a/_config/network/polkadot.toml b/_config/network/polkadot.toml index dd251135..ba476069 100644 --- a/_config/network/polkadot.toml +++ b/_config/network/polkadot.toml @@ -18,4 +18,7 @@ start_block_number = 2_700_565 data_endpoint = "https://nodes.web3.foundation/api/cohort/1/polkadot" [notification_generator] -unclaimed_payout_check_delay_hours = 9 \ No newline at end of file +unclaimed_payout_check_delay_hours = 9 + +[session_validator_performance_updater] +start_session_index = 5000 \ No newline at end of file diff --git a/_config/network/westend.toml b/_config/network/westend.toml index b3746e49..37d2b207 100644 --- a/_config/network/westend.toml +++ b/_config/network/westend.toml @@ -17,4 +17,7 @@ start_block_number = 10_130_000 [telemetry] # W3F wss://telemetry-backend.w3f.community/feed # Polkadot wss://feed.telemetry.polkadot.io/feed -websocket_url = "wss://feed.telemetry.polkadot.io/feed" \ No newline at end of file +websocket_url = "wss://feed.telemetry.polkadot.io/feed" + +[session_validator_performance_updater] +start_session_index = 5000 \ No newline at end of file diff --git a/_docker/compose/.env.sample b/_docker/compose/.env.sample index 848cc073..93d898c9 100644 --- a/_docker/compose/.env.sample +++ b/_docker/compose/.env.sample @@ -1,4 +1,4 @@ -VERSION=0.21.4 +VERSION=0.22.0 ENV=production LOG_LEVEL=debug @@ -83,3 +83,7 @@ FONT_DIR=/path/to/the/fonts/dir TMP_DIR=/path/to/temporary/files/dir PROMETHEUS_DIR=/path/to/the/prometheus/files/dir TEMPLATE_DIR=/path/to/the/templates/dir + +# SESSION VALIDATOR PERFORMANCE UPDATER +KUSAMA_SESSION_VALIDATOR_PERFORMANCE_UPDATER_START_SESSION_INDEX=5000 +POLKADOT_SESSION_VALIDATOR_PERFORMANCE_UPDATER_START_SESSION_INDEX=5000 \ No newline at end of file diff --git a/_docker/compose/05-docker-compose-subvt-kusama-services.yml b/_docker/compose/05-docker-compose-subvt-kusama-services.yml index 98abd76a..035b9bc0 100644 --- a/_docker/compose/05-docker-compose-subvt-kusama-services.yml +++ b/_docker/compose/05-docker-compose-subvt-kusama-services.yml @@ -298,6 +298,26 @@ services: - SUBVT__NETWORK_POSTGRES__HOST=subvt_kusama_postgres # kline updater - SUBVT__KLINE_UPDATER__TMP_DIR_PATH=/subvt/tmp + subvt_session_validator_performance_updater: + container_name: subvt_session_validator_performance_updater + restart: unless-stopped + image: "helikon/subvt-session-validator-performance-updater:${VERSION}" + networks: + - subvt_kusama + volumes: + - ${TMP_DIR}:/subvt/tmp + environment: + - SUBVT_ENV=${ENV} + - SUBVT_NETWORK=kusama + - SUBVT_CONFIG_DIR=/subvt/config + # log level + - SUBVT__LOG__SUBVT_LEVEL=${LOG_LEVEL} + # metrics + - SUBVT__METRICS__HOST=0.0.0.0 + # network postgres + - SUBVT__NETWORK_POSTGRES__HOST=subvt_kusama_postgres + # session validator performance updater + - SUBVT__SESSION_VALIDATOR_PERFORMANCE_UPDATER__START_SESSION_INDEX=${KUSAMA_SESSION_VALIDATOR_PERFORMANCE_UPDATER_START_SESSION_INDEX} networks: subvt_kusama: name: subvt_kusama diff --git a/_docker/compose/06-docker-compose-subvt-polkadot-services.yml b/_docker/compose/06-docker-compose-subvt-polkadot-services.yml index a8102a87..0203fa54 100644 --- a/_docker/compose/06-docker-compose-subvt-polkadot-services.yml +++ b/_docker/compose/06-docker-compose-subvt-polkadot-services.yml @@ -298,6 +298,26 @@ services: - SUBVT__NETWORK_POSTGRES__HOST=subvt_polkadot_postgres # kline updater - SUBVT__KLINE_UPDATER__TMP_DIR_PATH=/subvt/tmp + subvt_session_validator_performance_updater: + container_name: subvt_session_validator_performance_updater + restart: unless-stopped + image: "helikon/subvt-session-validator-performance-updater:${VERSION}" + networks: + - subvt_polkadot + volumes: + - ${TMP_DIR}:/subvt/tmp + environment: + - SUBVT_ENV=${ENV} + - SUBVT_NETWORK=polkadot + - SUBVT_CONFIG_DIR=/subvt/config + # log level + - SUBVT__LOG__SUBVT_LEVEL=${LOG_LEVEL} + # metrics + - SUBVT__METRICS__HOST=0.0.0.0 + # network postgres + - SUBVT__NETWORK_POSTGRES__HOST=subvt_polkadot_postgres + # session validator performance updater + - SUBVT__SESSION_VALIDATOR_PERFORMANCE_UPDATER__START_SESSION_INDEX=${POLKADOT_SESSION_VALIDATOR_PERFORMANCE_UPDATER_START_SESSION_INDEX} networks: subvt_polkadot: name: subvt_polkadot diff --git a/_docker/docker-build-and-publish.sh b/_docker/docker-build-and-publish.sh index 92dade83..66182dcf 100755 --- a/_docker/docker-build-and-publish.sh +++ b/_docker/docker-build-and-publish.sh @@ -74,4 +74,7 @@ docker build -t helikon/subvt-referendum-updater:"$1" -t helikon/subvt-referendu docker push --all-tags helikon/subvt-referendum-updater # kline updater docker build -t helikon/subvt-kline-updater:"$1" -t helikon/subvt-kline-updater:latest --no-cache --build-arg version="$1" -f ./network/15-subvt-kline-updater.dockerfile .. -docker push --all-tags helikon/subvt-kline-updater \ No newline at end of file +docker push --all-tags helikon/subvt-kline-updater +# session validator performance updater +docker build -t helikon/subvt-session-validator-performance-updater:"$1" -t helikon/subvt-session-validator-performance-updater:latest --no-cache --build-arg version="$1" -f ./network/16-subvt-session-validator-performance-updater.dockerfile .. +docker push --all-tags helikon/subvt-session-validator-performance-updater \ No newline at end of file diff --git a/_docker/docker-build.sh b/_docker/docker-build.sh index 2d25ad77..4351e12b 100755 --- a/_docker/docker-build.sh +++ b/_docker/docker-build.sh @@ -56,4 +56,6 @@ docker build -t helikon/subvt-report-service:"$1" --no-cache --build-arg version # referendum updater docker build -t helikon/subvt-referendum-updater:"$1" --no-cache --build-arg version="$1" -f ./network/14-subvt-referendum-updater.dockerfile .. # kline updater -docker build -t helikon/subvt-kline-updater:"$1" --no-cache --build-arg version="$1" -f ./network/15-subvt-kline-updater.dockerfile .. \ No newline at end of file +docker build -t helikon/subvt-kline-updater:"$1" --no-cache --build-arg version="$1" -f ./network/15-subvt-kline-updater.dockerfile .. +# session validator performance updater +docker build -t helikon/subvt-session-validator-performance-updater:"$1" --no-cache --build-arg version="$1" -f ./network/16-subvt-session-validator-performance-updater.dockerfile .. \ No newline at end of file diff --git a/_docker/network/16-subvt-session-validator-performance-updater.dockerfile b/_docker/network/16-subvt-session-validator-performance-updater.dockerfile new file mode 100644 index 00000000..ff9ebb19 --- /dev/null +++ b/_docker/network/16-subvt-session-validator-performance-updater.dockerfile @@ -0,0 +1,7 @@ +ARG version +FROM helikon/subvt-backend-lib:$version as builder + +FROM helikon/subvt-backend-base:$version +# copy executable +COPY --from=builder /subvt/bin/subvt-session-validator-performance-updater /usr/local/bin/ +CMD ["subvt-session-validator-performance-updater"] \ No newline at end of file diff --git a/_migrations/network/migrations/20241127061013_session_validator_performance.down.sql b/_migrations/network/migrations/20241127061013_session_validator_performance.down.sql new file mode 100644 index 00000000..d2f607c5 --- /dev/null +++ b/_migrations/network/migrations/20241127061013_session_validator_performance.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/_migrations/network/migrations/20241127061013_session_validator_performance.up.sql b/_migrations/network/migrations/20241127061013_session_validator_performance.up.sql new file mode 100644 index 00000000..e296a102 --- /dev/null +++ b/_migrations/network/migrations/20241127061013_session_validator_performance.up.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS sub_session_validator_performance +( + id SERIAL PRIMARY KEY, + validator_account_id VARCHAR(66) NOT NULL, + era_index BIGINT NOT NULL, + session_index BIGINT NOT NULL, + active_validator_index BIGINT NOT NULL, + authored_block_count INT NOT NULL DEFAULT 0, + para_validator_group_index BIGINT, + para_validator_index BIGINT, + implicit_attestation_count INT, + explicit_attestation_count INT, + missed_attestation_count INT, + attestations_per_billion INT, + created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS sub_session_validator_performance_idx_validator_account_id + ON sub_session_validator_performance (validator_account_id); + CREATE INDEX IF NOT EXISTS sub_session_validator_performance_idx_session_index + ON sub_session_validator_performance (session_index); +CREATE INDEX IF NOT EXISTS sub_session_validator_performance_idx_validator_account_id_session_index_desc + ON sub_session_validator_performance (validator_account_id, session_index DESC); +CREATE INDEX IF NOT EXISTS sub_session_validator_performance_idx_validator_account_id_active_validator_index_session_index_desc + ON sub_session_validator_performance (validator_account_id, active_validator_index, session_index DESC); +CREATE INDEX IF NOT EXISTS sub_session_validator_performance_idx_validator_account_id_para_validator_index_session_index_desc + ON sub_session_validator_performance (validator_account_id, para_validator_index, session_index DESC); \ No newline at end of file diff --git a/subvt-app-service/Cargo.toml b/subvt-app-service/Cargo.toml index 19e747c4..0529ba30 100644 --- a/subvt-app-service/Cargo.toml +++ b/subvt-app-service/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-app-service" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-block-processor/Cargo.toml b/subvt-block-processor/Cargo.toml index cca09f90..f87a98ff 100644 --- a/subvt-block-processor/Cargo.toml +++ b/subvt-block-processor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-block-processor" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-block-processor/src/lib.rs b/subvt-block-processor/src/lib.rs index 67fd8b60..e12d7302 100644 --- a/subvt-block-processor/src/lib.rs +++ b/subvt-block-processor/src/lib.rs @@ -494,6 +494,10 @@ impl BlockProcessor { } } } + // save disputes + for statement_set in votes.disputes { + for _statement in statement_set.statements {} + } log::debug!( "Processed {} para votes for {} paras.", total_vote_count, diff --git a/subvt-config/Cargo.toml b/subvt-config/Cargo.toml index a60279c5..06381eca 100644 --- a/subvt-config/Cargo.toml +++ b/subvt-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-config" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-config/src/lib.rs b/subvt-config/src/lib.rs index 56c0dc48..40eb41d9 100644 --- a/subvt-config/src/lib.rs +++ b/subvt-config/src/lib.rs @@ -235,6 +235,7 @@ pub struct MetricsConfig { pub app_service_port: u16, pub referendum_updater_port: u16, pub kline_updater_port: u16, + pub session_validator_performance_updater_port: u16, } /// Plotter config. @@ -274,6 +275,12 @@ pub struct KLineUpdaterConfig { pub begin_day: u32, } +#[derive(Clone, Debug, Deserialize)] +pub struct SessionValidatorPerformanceUpdaterConfig { + pub start_session_index: u64, + pub sleep_seconds: u64, +} + /// Whole configuration. #[derive(Clone, Debug, Deserialize)] pub struct Config { @@ -300,6 +307,7 @@ pub struct Config { pub app_service: AppServiceConfig, pub referendum_updater: ReferendumUpdaterConfig, pub kline_updater: KLineUpdaterConfig, + pub session_validator_performance_updater: SessionValidatorPerformanceUpdaterConfig, } impl Config { diff --git a/subvt-governance/Cargo.toml b/subvt-governance/Cargo.toml index e5e8db54..dde44917 100644 --- a/subvt-governance/Cargo.toml +++ b/subvt-governance/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-governance" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-kline-updater/Cargo.toml b/subvt-kline-updater/Cargo.toml index daf335f1..20a2d704 100644 --- a/subvt-kline-updater/Cargo.toml +++ b/subvt-kline-updater/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-kline-updater" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-logging/Cargo.toml b/subvt-logging/Cargo.toml index 5431a801..648bd8e8 100644 --- a/subvt-logging/Cargo.toml +++ b/subvt-logging/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-logging" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-logging/src/lib.rs b/subvt-logging/src/lib.rs index 271767c9..8722a159 100644 --- a/subvt-logging/src/lib.rs +++ b/subvt-logging/src/lib.rs @@ -29,6 +29,10 @@ pub fn init(config: &subvt_config::Config) { builder.filter(Some("subvt_plotter"), log_level); builder.filter(Some("subvt_referendum_updater"), log_level); builder.filter(Some("subvt_report_service"), log_level); + builder.filter( + Some("subvt_session_validator_performance_updater"), + log_level, + ); builder.filter(Some("subvt_substrate_client"), log_level); builder.filter(Some("subvt_telegram_bot"), log_level); builder.filter(Some("subvt_telemetry_processor"), log_level); diff --git a/subvt-metrics/Cargo.toml b/subvt-metrics/Cargo.toml index 5360d7f4..11c896c7 100644 --- a/subvt-metrics/Cargo.toml +++ b/subvt-metrics/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-metrics" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-network-status-server/Cargo.toml b/subvt-network-status-server/Cargo.toml index 47c074fe..db8c202a 100644 --- a/subvt-network-status-server/Cargo.toml +++ b/subvt-network-status-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-network-status-server" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-network-status-updater/Cargo.toml b/subvt-network-status-updater/Cargo.toml index a3b36a59..7a928947 100644 --- a/subvt-network-status-updater/Cargo.toml +++ b/subvt-network-status-updater/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-network-status-updater" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-nft/Cargo.toml b/subvt-nft/Cargo.toml index fa4e1a71..65f7562e 100644 --- a/subvt-nft/Cargo.toml +++ b/subvt-nft/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-nft" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-notification-generator/Cargo.toml b/subvt-notification-generator/Cargo.toml index fd4dc53d..c105a7cc 100644 --- a/subvt-notification-generator/Cargo.toml +++ b/subvt-notification-generator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-notification-generator" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-notification-processor/Cargo.toml b/subvt-notification-processor/Cargo.toml index 69365690..b4fcb8b8 100644 --- a/subvt-notification-processor/Cargo.toml +++ b/subvt-notification-processor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-notification-processor" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-onekv-updater/Cargo.toml b/subvt-onekv-updater/Cargo.toml index d132b455..32c16be8 100644 --- a/subvt-onekv-updater/Cargo.toml +++ b/subvt-onekv-updater/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-onekv-updater" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-persistence/Cargo.toml b/subvt-persistence/Cargo.toml index f08d3a20..cf2e86b0 100644 --- a/subvt-persistence/Cargo.toml +++ b/subvt-persistence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-persistence" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-persistence/src/postgres/network/epoch.rs b/subvt-persistence/src/postgres/network/epoch.rs index 37554394..2419b269 100644 --- a/subvt-persistence/src/postgres/network/epoch.rs +++ b/subvt-persistence/src/postgres/network/epoch.rs @@ -15,6 +15,7 @@ impl PostgreSQLNetworkStorage { .await?; Ok(maybe_db_epoch.map(|db_epoch| Epoch { index: db_epoch.0 as u64, + era_index: db_epoch.1 as u32, start_block_number: db_epoch.2 as u32, start_timestamp: db_epoch.3 as u64, end_timestamp: db_epoch.4 as u64, @@ -34,6 +35,7 @@ impl PostgreSQLNetworkStorage { .await?; Ok(maybe_db_epoch.map(|db_epoch| Epoch { index: db_epoch.0 as u64, + era_index: db_epoch.1 as u32, start_block_number: db_epoch.2 as u32, start_timestamp: db_epoch.3 as u64, end_timestamp: db_epoch.4 as u64, diff --git a/subvt-persistence/src/postgres/network/mod.rs b/subvt-persistence/src/postgres/network/mod.rs index a78674ac..ea3e04a4 100644 --- a/subvt-persistence/src/postgres/network/mod.rs +++ b/subvt-persistence/src/postgres/network/mod.rs @@ -17,6 +17,7 @@ pub mod nft; pub mod notify; pub mod onekv; pub mod para; +pub mod performance; pub mod referendum; pub mod report; pub mod staking; diff --git a/subvt-persistence/src/postgres/network/performance.rs b/subvt-persistence/src/postgres/network/performance.rs new file mode 100644 index 00000000..c63d2600 --- /dev/null +++ b/subvt-persistence/src/postgres/network/performance.rs @@ -0,0 +1,17 @@ +use crate::postgres::network::PostgreSQLNetworkStorage; + +impl PostgreSQLNetworkStorage { + pub async fn get_session_validator_performance_updater_last_processed_session_id( + &self, + ) -> anyhow::Result> { + let row: (Option,) = sqlx::query_as( + r#" + SELECT MAX(session_index) + FROM sub_session_validator_performance + "#, + ) + .fetch_one(&self.connection_pool) + .await?; + Ok(row.0.map(|index| index as u64)) + } +} diff --git a/subvt-plotter/Cargo.toml b/subvt-plotter/Cargo.toml index 1c9068bb..b57238b1 100644 --- a/subvt-plotter/Cargo.toml +++ b/subvt-plotter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-plotter" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-proc-macro/Cargo.toml b/subvt-proc-macro/Cargo.toml index aff9485a..958c5883 100644 --- a/subvt-proc-macro/Cargo.toml +++ b/subvt-proc-macro/Cargo.toml @@ -3,7 +3,7 @@ proc-macro = true [package] name = "subvt-proc-macro" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-referendum-updater/Cargo.toml b/subvt-referendum-updater/Cargo.toml index 60e833f6..45c4e267 100644 --- a/subvt-referendum-updater/Cargo.toml +++ b/subvt-referendum-updater/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-referendum-updater" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-report-service/Cargo.toml b/subvt-report-service/Cargo.toml index 6418aaae..a1d6522f 100644 --- a/subvt-report-service/Cargo.toml +++ b/subvt-report-service/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-report-service" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-report-service/src/session/validator.rs b/subvt-report-service/src/session/validator.rs index 90192a09..ed6c6c6f 100644 --- a/subvt-report-service/src/session/validator.rs +++ b/subvt-report-service/src/session/validator.rs @@ -130,17 +130,10 @@ async fn get_session_validator_report( None => return Ok(Some(report)), } // get votes (summary) - let mut para_votes_summary = ParaVotesSummary::default(); let votes = postgres .get_session_para_validator_votes(session_index, report.para_validator_index.unwrap()) .await?; - for vote in votes { - match vote.vote { - ParaVoteType::EXPLICIT => para_votes_summary.explicit += 1, - ParaVoteType::IMPLICIT => para_votes_summary.implicit += 1, - ParaVoteType::MISSED => para_votes_summary.missed += 1, - } - } + let para_votes_summary = ParaVotesSummary::from_para_votes(&votes); report.para_votes_summary = Some(para_votes_summary); Ok(Some(report)) } diff --git a/subvt-service-common/Cargo.toml b/subvt-service-common/Cargo.toml index f06a2c9c..8d003bf3 100644 --- a/subvt-service-common/Cargo.toml +++ b/subvt-service-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-service-common" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-session-validator-performance-updater/Cargo.toml b/subvt-session-validator-performance-updater/Cargo.toml new file mode 100644 index 00000000..a19ee051 --- /dev/null +++ b/subvt-session-validator-performance-updater/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "subvt-session-validator-performance-updater" +version = "0.22.0" +edition = "2021" +rust-version = "1.67.0" + +[dependencies] +anyhow = { workspace = true } +async-lock = "3.4" +async-trait = "0.1" +lazy_static = { workspace = true } +log = { workspace = true } +once_cell = "1" +subvt-config = { path = "../subvt-config" } +subvt-logging = { path = "../subvt-logging" } +subvt-metrics = { path = "../subvt-metrics" } +subvt-persistence = { path = "../subvt-persistence" } +subvt-service-common = { path = "../subvt-service-common" } +subvt-types = { path = "../subvt-types" } +tokio = { version = "1.41", features = ["full"] } diff --git a/subvt-session-validator-performance-updater/src/lib.rs b/subvt-session-validator-performance-updater/src/lib.rs new file mode 100644 index 00000000..cf2d391f --- /dev/null +++ b/subvt-session-validator-performance-updater/src/lib.rs @@ -0,0 +1,157 @@ +//! Updates the complete 1KV data for the network (only Polkadot and Kusama) on the database. +#![warn(clippy::disallowed_types)] + +use async_trait::async_trait; +use lazy_static::lazy_static; +use std::cmp::max; +use subvt_config::Config; +use subvt_persistence::postgres::network::PostgreSQLNetworkStorage; +use subvt_service_common::Service; +use subvt_types::performance::SessionValidatorPerformance; +use subvt_types::report::ParaVotesSummary; +use subvt_types::substrate::Epoch; + +mod metrics; + +lazy_static! { + static ref CONFIG: Config = Config::default(); +} + +pub struct SessionValidatorPerformanceUpdater; + +impl SessionValidatorPerformanceUpdater { + async fn process_session( + &self, + postgres: &PostgreSQLNetworkStorage, + session: &Epoch, + ) -> anyhow::Result<()> { + let active_validator_account_ids = postgres + .get_era_validator_account_ids(session.era_index, true) + .await?; + log::info!( + "Process {} active validators in session {}.", + active_validator_account_ids.len(), + session.index + ); + let mut era_active_validators = Vec::new(); + for account_id in active_validator_account_ids.iter() { + let era_validator = match postgres + .get_era_validator_by_session_index(account_id, session.index) + .await? + { + Some(era_validator) => era_validator, + _ => continue, + }; + era_active_validators.push(era_validator); + } + let mut session_validator_performances = Vec::new(); + for era_active_validator in era_active_validators.iter() { + let mut performance = SessionValidatorPerformance { + validator_account_id: era_active_validator.validator_account_id, + era_index: session.era_index, + session_index: session.index, + active_validator_index: era_active_validator.active_validator_index.unwrap(), + ..Default::default() + }; + // para-related + let maybe_para_validator = postgres + .get_session_para_validator( + session.index, + &era_active_validator.validator_account_id, + ) + .await?; + if let Some(para_validator) = maybe_para_validator { + performance.para_validator_group_index = + Some(para_validator.para_validator_group_index); + performance.para_validator_index = Some(para_validator.para_validator_index); + let votes = postgres + .get_session_para_validator_votes( + session.index, + para_validator.para_validator_index, + ) + .await?; + let votes_summary = ParaVotesSummary::from_para_votes(&votes); + performance.implicit_attestation_count = Some(votes_summary.implicit); + performance.explicit_attestation_count = Some(votes_summary.explicit); + performance.missed_attestation_count = Some(votes_summary.missed); + let attestation_count = votes_summary.implicit + votes_summary.explicit; + let total_attestation_slots = + votes_summary.implicit + votes_summary.explicit + votes_summary.missed; + let attestations_per_billion = + (attestation_count / total_attestation_slots) * 1_000_000_000; + performance.attestations_per_billion = Some(attestations_per_billion); + } + session_validator_performances.push(performance); + } + // persist performances & update state + log::info!( + "Persist {} validator performances for session {}.", + session_validator_performances.len(), + session.index + ); + Ok(()) + } +} + +#[async_trait(?Send)] +impl Service for SessionValidatorPerformanceUpdater { + fn get_metrics_server_addr() -> (&'static str, u16) { + ( + CONFIG.metrics.host.as_str(), + CONFIG.metrics.session_validator_performance_updater_port, + ) + } + + async fn run(&'static self) -> anyhow::Result<()> { + log::info!( + "Session validator performance updater has started with {} seconds refresh wait period.", + CONFIG.session_validator_performance_updater.sleep_seconds, + ); + let postgres = + PostgreSQLNetworkStorage::new(&CONFIG, CONFIG.get_network_postgres_url()).await?; + loop { + let last_processed_session_index = postgres + .get_session_validator_performance_updater_last_processed_session_id() + .await? + .unwrap_or(0); + let current_session_index = postgres + .get_current_epoch() + .await? + .map(|epoch| epoch.index) + .unwrap_or(1); + let start_session_index = max( + last_processed_session_index, + CONFIG + .session_validator_performance_updater + .start_session_index, + ); + log::info!( + "Process sessions {}-{}.", + start_session_index, + (current_session_index - 1) + ); + if start_session_index >= (current_session_index - 1) { + log::warn!( + "Start session index is greater than or equal to end session index. No-op." + ); + } else { + for session_index in start_session_index..current_session_index { + log::info!("Process session {session_index}."); + let session = match postgres.get_epoch_by_index(session_index).await? { + Some(session) => session, + _ => { + log::warn!("Session {session_index} not found in storage. Continue with the next session."); + continue; + } + }; + self.process_session(&postgres, &session).await?; + } + } + log::info!( + "Sleep for {} seconds.", + CONFIG.session_validator_performance_updater.sleep_seconds + ); + tokio::time::sleep(std::time::Duration::from_secs(CONFIG.dn.refresh_seconds)).await; + } + } +} diff --git a/subvt-session-validator-performance-updater/src/main.rs b/subvt-session-validator-performance-updater/src/main.rs new file mode 100644 index 00000000..69ea1f6c --- /dev/null +++ b/subvt-session-validator-performance-updater/src/main.rs @@ -0,0 +1,12 @@ +use lazy_static::lazy_static; +use subvt_service_common::Service; +use subvt_session_validator_performance_updater::SessionValidatorPerformanceUpdater; + +lazy_static! { + static ref SERVICE: SessionValidatorPerformanceUpdater = SessionValidatorPerformanceUpdater; +} + +#[tokio::main] +async fn main() { + SERVICE.start().await; +} diff --git a/subvt-session-validator-performance-updater/src/metrics.rs b/subvt-session-validator-performance-updater/src/metrics.rs new file mode 100644 index 00000000..0abc1f32 --- /dev/null +++ b/subvt-session-validator-performance-updater/src/metrics.rs @@ -0,0 +1,16 @@ +use once_cell::sync::Lazy; +use subvt_metrics::registry::IntGauge; + +const _METRIC_PREFIX: &str = "subvt_validator_score_updater"; + +pub fn _target_finalized_block_number() -> IntGauge { + static METER: Lazy = Lazy::new(|| { + subvt_metrics::registry::register_int_gauge( + _METRIC_PREFIX, + "target_finalized_block_number", + "Number of the target finalized block on the node", + ) + .unwrap() + }); + METER.clone() +} diff --git a/subvt-substrate-client/Cargo.toml b/subvt-substrate-client/Cargo.toml index f04095b5..77d45173 100644 --- a/subvt-substrate-client/Cargo.toml +++ b/subvt-substrate-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-substrate-client" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-substrate-client/src/lib.rs b/subvt-substrate-client/src/lib.rs index d5ccfd1a..e2a18d4c 100644 --- a/subvt-substrate-client/src/lib.rs +++ b/subvt-substrate-client/src/lib.rs @@ -252,6 +252,7 @@ impl SubstrateClient { /// Get current epoch at the given block. pub async fn get_current_epoch(&self, block_hash: &str) -> anyhow::Result { let index = self.get_current_epoch_index(block_hash).await?; + let era = self.get_active_era(block_hash).await?; let start_block_number = { let hex_string: String = self .ws_client @@ -267,6 +268,7 @@ impl SubstrateClient { let end_timestamp = start_timestamp + get_metadata_epoch_duration_millis(&self.metadata)?; Ok(Epoch { index, + era_index: era.index, start_block_number, start_timestamp, end_timestamp, diff --git a/subvt-telegram-bot/Cargo.toml b/subvt-telegram-bot/Cargo.toml index e8240db3..45283e99 100644 --- a/subvt-telegram-bot/Cargo.toml +++ b/subvt-telegram-bot/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-telegram-bot" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-telemetry-processor/Cargo.toml b/subvt-telemetry-processor/Cargo.toml index da76331a..b31ba6f0 100644 --- a/subvt-telemetry-processor/Cargo.toml +++ b/subvt-telemetry-processor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-telemetry-processor" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-types/Cargo.toml b/subvt-types/Cargo.toml index 15d588d1..eba815e3 100644 --- a/subvt-types/Cargo.toml +++ b/subvt-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-types" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-types/src/lib.rs b/subvt-types/src/lib.rs index 09ec3d10..00cade7e 100644 --- a/subvt-types/src/lib.rs +++ b/subvt-types/src/lib.rs @@ -6,6 +6,7 @@ pub mod dn; pub mod err; pub mod governance; pub mod kline; +pub mod performance; pub mod rdb; pub mod report; pub mod sub_id; diff --git a/subvt-types/src/performance.rs b/subvt-types/src/performance.rs new file mode 100644 index 00000000..15a34c53 --- /dev/null +++ b/subvt-types/src/performance.rs @@ -0,0 +1,17 @@ +use crate::crypto::AccountId; + +#[derive(Default)] +pub struct SessionValidatorPerformance { + pub id: u64, + pub validator_account_id: AccountId, + pub era_index: u32, + pub session_index: u64, + pub active_validator_index: u64, + pub authored_block_count: u32, + pub para_validator_group_index: Option, + pub para_validator_index: Option, + pub implicit_attestation_count: Option, + pub explicit_attestation_count: Option, + pub missed_attestation_count: Option, + pub attestations_per_billion: Option, +} diff --git a/subvt-types/src/report.rs b/subvt-types/src/report.rs index fdcf41e2..ca91b75e 100644 --- a/subvt-types/src/report.rs +++ b/subvt-types/src/report.rs @@ -84,6 +84,20 @@ pub struct ParaVotesSummary { pub missed: u32, } +impl ParaVotesSummary { + pub fn from_para_votes(para_votes: &[ParaVote]) -> ParaVotesSummary { + let mut para_votes_summary = ParaVotesSummary::default(); + for vote in para_votes { + match vote.vote { + ParaVoteType::EXPLICIT => para_votes_summary.explicit += 1, + ParaVoteType::IMPLICIT => para_votes_summary.implicit += 1, + ParaVoteType::MISSED => para_votes_summary.missed += 1, + } + } + para_votes_summary + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub enum ParaVoteType { #[serde(rename = "explicit")] diff --git a/subvt-types/src/substrate/mod.rs b/subvt-types/src/substrate/mod.rs index 7d52ae6a..577b7809 100644 --- a/subvt-types/src/substrate/mod.rs +++ b/subvt-types/src/substrate/mod.rs @@ -401,6 +401,7 @@ impl Era { #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct Epoch { pub index: u64, + pub era_index: u32, pub start_block_number: u32, pub start_timestamp: u64, pub end_timestamp: u64, diff --git a/subvt-utility/Cargo.toml b/subvt-utility/Cargo.toml index 8acb0856..8a555b7c 100644 --- a/subvt-utility/Cargo.toml +++ b/subvt-utility/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-utility" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-validator-details-server/Cargo.toml b/subvt-validator-details-server/Cargo.toml index d08f9f72..84949afa 100644 --- a/subvt-validator-details-server/Cargo.toml +++ b/subvt-validator-details-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-validator-details-server" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-validator-list-server/Cargo.toml b/subvt-validator-list-server/Cargo.toml index bf13b354..71a746da 100644 --- a/subvt-validator-list-server/Cargo.toml +++ b/subvt-validator-list-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-validator-list-server" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0" diff --git a/subvt-validator-list-updater/Cargo.toml b/subvt-validator-list-updater/Cargo.toml index 5b2c9489..f11121cc 100644 --- a/subvt-validator-list-updater/Cargo.toml +++ b/subvt-validator-list-updater/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "subvt-validator-list-updater" -version = "0.21.4" +version = "0.22.0" edition = "2021" rust-version = "1.67.0"