From 59c242f3bc0dcac93753d2930e3ab458bc70bc87 Mon Sep 17 00:00:00 2001 From: anonymousGiga Date: Sat, 9 Dec 2023 13:18:10 +0800 Subject: [PATCH] Perf: Add metrics to facilitate performance testing. --- Cargo.lock | 120 ++++++++++++++++-- Cargo.toml | 11 +- bin/reth/Cargo.toml | 38 +++++- bin/reth/src/builder.rs | 11 ++ bin/reth/src/cli/mod.rs | 3 + bin/reth/src/commands/stage/mod.rs | 11 ++ bin/reth/src/commands/stage/run.rs | 21 +++ crates/revm/Cargo.toml | 15 +++ crates/revm/src/processor.rs | 28 ++++ crates/stages/Cargo.toml | 26 ++++ crates/stages/src/pipeline/builder.rs | 14 ++ crates/stages/src/stages/execution.rs | 31 ++++- crates/storage/provider/Cargo.toml | 7 + .../bundle_state_with_receipts.rs | 13 ++ .../src/bundle_state/state_changes.rs | 18 +++ .../src/bundle_state/state_reverts.rs | 13 ++ .../storage/provider/src/traits/executor.rs | 6 + 17 files changed, 369 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 640169a773954..5d752b1c102d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,6 +1396,17 @@ dependencies = [ "reth", ] +[[package]] +name = "coarsetime" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b3839cf01bb7960114be3ccf2340f541b6d0c81f8690b007b2b39f750f7e5d" +dependencies = [ + "libc", + "wasix", + "wasm-bindgen", +] + [[package]] name = "cobs" version = "0.2.3" @@ -1747,6 +1758,26 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ctor" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" +dependencies = [ + "quote", + "syn 2.0.48", +] + [[package]] name = "ctr" version = "0.7.0" @@ -3361,7 +3392,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -4408,6 +4439,17 @@ dependencies = [ "adler", ] +[[package]] +name = "minstant" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e326a3745e3f61d1715722ed4e72c0e9689aac76f1496c4deeeb64dd68f93fb" +dependencies = [ + "coarsetime", + "ctor 0.1.26", + "web-time", +] + [[package]] name = "mio" version = "0.8.10" @@ -4922,6 +4964,18 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "perf-metrics" +version = "0.1.0" +source = "git+https://github.com/megaeth-labs/reth-perf-utils.git#464994aada3219baf59ede529221a3c48f31e94e" +dependencies = [ + "ctor 0.2.6", + "minstant", + "revm", + "revm-utils", + "tokio", +] + [[package]] name = "pest" version = "2.7.7" @@ -5713,7 +5767,9 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-process", "metrics-util", + "minstant", "once_cell", + "perf-metrics", "pin-project", "proptest", "rand 0.8.5", @@ -5757,6 +5813,7 @@ dependencies = [ "reth-tracing", "reth-transaction-pool", "reth-trie", + "revm", "revm-inspectors", "secp256k1 0.27.0", "serde", @@ -6617,6 +6674,7 @@ dependencies = [ "itertools 0.12.1", "metrics", "parking_lot 0.12.1", + "perf-metrics", "pin-project", "rand 0.8.5", "rayon", @@ -6662,6 +6720,7 @@ dependencies = [ name = "reth-revm" version = "0.1.0-alpha.18" dependencies = [ + "perf-metrics", "reth-consensus-common", "reth-interfaces", "reth-node-api", @@ -6895,6 +6954,7 @@ dependencies = [ "metrics", "num-traits", "paste", + "perf-metrics", "pin-project", "pprof", "rand 0.8.5", @@ -7023,13 +7083,14 @@ dependencies = [ [[package]] name = "revm" version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8154cec6f8c4f543a379cf00a839cf47c67f405e8a92361a7791c55196c9b7e2" +source = "git+https://github.com/megaeth-labs/revm.git?branch=andy/debug/make-utils-independent#18d03d3e52d1093d308db05f9888ef5ac22f714e" dependencies = [ "auto_impl", "cfg-if", + "hashbrown 0.14.3", "revm-interpreter", "revm-precompile", + "revm-utils", "serde", "serde_json", ] @@ -7037,7 +7098,7 @@ dependencies = [ [[package]] name = "revm-inspectors" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/evm-inspectors?rev=ac63f06#ac63f069977d04cd68d96fe7ad9dd4b74ceab44e" +source = "git+https://github.com/megaeth-labs/evm-inspectors.git?branch=make-utils-independent#6d447c3c6368bcf7635662ca543c87f1556e597d" dependencies = [ "alloy-primitives", "alloy-rpc-trace-types", @@ -7056,18 +7117,17 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff72cc825a9c44b28749a6181e21fa85ba45ac8d4b5732a7ded165a770ecbd9" +source = "git+https://github.com/megaeth-labs/revm.git?branch=andy/debug/make-utils-independent#18d03d3e52d1093d308db05f9888ef5ac22f714e" dependencies = [ "revm-primitives", + "revm-utils", "serde", ] [[package]] name = "revm-precompile" version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186373108c0a46e47368752372cc721d4b0ab4886d6786d180ef9dbf4dd71865" +source = "git+https://github.com/megaeth-labs/revm.git?branch=andy/debug/make-utils-independent#18d03d3e52d1093d308db05f9888ef5ac22f714e" dependencies = [ "aurora-engine-modexp", "blst", @@ -7084,8 +7144,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093dc5df253eececaf03ffe0c12d3fd7ce5dea3342bc9de109d1aac2647ffa81" +source = "git+https://github.com/megaeth-labs/revm.git?branch=andy/debug/make-utils-independent#18d03d3e52d1093d308db05f9888ef5ac22f714e" dependencies = [ "alloy-primitives", "auto_impl", @@ -7099,7 +7158,20 @@ dependencies = [ "hashbrown 0.14.3", "hex", "once_cell", + "revm-utils", + "serde", +] + +[[package]] +name = "revm-utils" +version = "0.1.0" +source = "git+https://github.com/megaeth-labs/reth-perf-utils.git#464994aada3219baf59ede529221a3c48f31e94e" +dependencies = [ + "allocator-api2", + "ctor 0.2.6", "serde", + "serde_arrays", + "serde_json", ] [[package]] @@ -7584,6 +7656,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_arrays" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38636132857f68ec3d5f3eb121166d2af33cb55174c4d5ff645db6165cbef0fd" +dependencies = [ + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.14" @@ -9161,6 +9242,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasix" +version = "0.12.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fbb4ef9bbca0c1170e0b00dd28abc9e3b68669821600cad1caaed606583c6d" +dependencies = [ + "wasi 0.11.0+wasi-snapshot-preview1", +] + [[package]] name = "wasm-bindgen" version = "0.2.91" @@ -9250,6 +9340,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee269d72cc29bf77a2c4bc689cc750fb39f5cbd493d2205bbb3f5c7779cf7b0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/Cargo.toml b/Cargo.toml index e6cf1626d808e..77bd13b2e2ca2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,9 +170,9 @@ reth-transaction-pool = { path = "crates/transaction-pool" } reth-trie = { path = "crates/trie" } # revm -revm = { version = "6.0", features = ["std", "secp256k1"], default-features = false } -revm-primitives = { version = "2.0", features = ["std"], default-features = false } -revm-inspectors = { git = "https://github.com/paradigmxyz/evm-inspectors", rev = "ac63f06" } +revm = { git = "https://github.com/megaeth-labs/revm.git", branch = "andy/debug/make-utils-independent", features = ["std", "secp256k1"], default-features = false } +revm-primitives = { git = "https://github.com/megaeth-labs/revm.git", branch = "andy/debug/make-utils-independent", features = ["std"], default-features = false } +revm-inspectors = { git = "https://github.com/megaeth-labs/evm-inspectors.git", branch = "make-utils-independent" } # eth alloy-chains = { version = "0.1", feature = ["serde", "rlp", "arbitrary"] } @@ -269,6 +269,9 @@ proptest-derive = "0.4" serial_test = "3" similar-asserts = "1.5.0" +## support perf test +perf-metrics = { git = "https://github.com/megaeth-labs/reth-perf-utils.git" } + [workspace.metadata.cargo-udeps.ignore] # ignored because this is mutually exclusive with the optimism payload builder via feature flags -normal = ["reth-ethereum-payload-builder", "reth-node-ethereum"] +normal = ["reth-ethereum-payload-builder", "reth-node-ethereum"] \ No newline at end of file diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 1d1d4639ff7d4..4efef85ece92a 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -60,6 +60,8 @@ reth-node-optimism = { workspace = true, optional = true, features = [ ] } reth-node-core.workspace = true +revm = { workspace = true, optional = true} + # crypto alloy-rlp.workspace = true alloy-chains.workspace = true @@ -88,6 +90,7 @@ reth-metrics.workspace = true metrics.workspace = true once_cell.workspace = true + # test vectors generation proptest.workspace = true rand.workspace = true @@ -124,6 +127,10 @@ itertools.workspace = true rayon.workspace = true boyer-moore-magiclen = "0.2.16" +minstant = { version = "0.1.3", optional = true } +perf-metrics = { workspace = true, optional = true } + + [target.'cfg(not(windows))'.dependencies] jemallocator = { version = "0.5.0", optional = true } @@ -168,6 +175,35 @@ optimism = [ # no-op feature flag for switching between the `optimism` and default functionality in CI matrices ethereum = [] +# support perf test +open_performance_dashboard = [ + "reth-stages/open_performance_dashboard", + "perf-metrics", +] +enable_opcode_metrics = [ + "perf-metrics/enable_opcode_metrics", + "reth-revm/enable_opcode_metrics", + "reth-stages/enable_opcode_metrics", + "open_performance_dashboard", +] +enable_cache_record = [ + "perf-metrics/enable_cache_record", + "reth-revm/enable_cache_record", + "reth-stages/enable_cache_record", + "open_performance_dashboard", +] +enable_tps_gas_record = [ + "perf-metrics/enable_tps_gas_record", + "reth-stages/enable_tps_gas_record", + "open_performance_dashboard", +] +finish_after_execution_stage = ["reth-stages/finish_after_execution_stage"] +enable_execution_duration_record = [ + "perf-metrics/enable_execution_duration_record", + "reth-stages/enable_execution_duration_record", + "reth-revm/enable_execution_duration_record", + "open_performance_dashboard", +] [[bin]] name = "reth" @@ -176,4 +212,4 @@ path = "src/main.rs" [[bin]] name = "op-reth" path = "src/optimism.rs" -required-features = ["optimism"] +required-features = ["optimism"] \ No newline at end of file diff --git a/bin/reth/src/builder.rs b/bin/reth/src/builder.rs index 3ea5f242d6547..d976f9eed1473 100644 --- a/bin/reth/src/builder.rs +++ b/bin/reth/src/builder.rs @@ -46,6 +46,9 @@ use std::{path::PathBuf, sync::Arc}; use tokio::sync::{mpsc::unbounded_channel, oneshot}; use tracing::*; +#[cfg(feature = "open_performance_dashboard")] +use perf_metrics::dashboard::DashboardListener; + /// Re-export `NodeConfig` from `reth_node_core`. pub use reth_node_core::node_config::NodeConfig; @@ -146,6 +149,14 @@ impl NodeBuilderWit info!(target: "reth::cli", "{}", DisplayHardforks::new(self.config.chain.hardforks())); + #[cfg(feature = "open_performance_dashboard")] + { + let (dashboard_tx, dashboard_rx) = unbounded_channel(); + let dashboard_listener = DashboardListener::new(dashboard_rx); + executor.spawn_critical("dashboard listener task", dashboard_listener); + perf_metrics::set_metric_event_sender(dashboard_tx); + } + let consensus = self.config.consensus(); debug!(target: "reth::cli", "Spawning stages metrics listener task"); diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 659e36bfea684..78774c028e63f 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -82,7 +82,10 @@ impl Cli { Commands::Init(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::Import(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute()), + #[cfg(not(feature = "open_performance_dashboard"))] Commands::Stage(command) => runner.run_blocking_until_ctrl_c(command.execute()), + #[cfg(feature = "open_performance_dashboard")] + Commands::Stage(command) => runner.run_command_until_exit(|ctx| command.execute(ctx)), Commands::P2P(command) => runner.run_until_ctrl_c(command.execute()), Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), Commands::Config(command) => runner.run_until_ctrl_c(command.execute()), diff --git a/bin/reth/src/commands/stage/mod.rs b/bin/reth/src/commands/stage/mod.rs index a3e2fa19ebbf9..bcab6ebfc2265 100644 --- a/bin/reth/src/commands/stage/mod.rs +++ b/bin/reth/src/commands/stage/mod.rs @@ -34,6 +34,7 @@ pub enum Subcommands { impl Command { /// Execute `stage` command + #[cfg(not(feature = "open_performance_dashboard"))] pub async fn execute(self) -> eyre::Result<()> { match self.command { Subcommands::Run(command) => command.execute().await, @@ -42,4 +43,14 @@ impl Command { Subcommands::Unwind(command) => command.execute().await, } } + /// Execute `stage` command + #[cfg(feature = "open_performance_dashboard")] + pub async fn execute(self, _ctx: crate::runner::CliContext) -> eyre::Result<()> { + match self.command { + Subcommands::Run(command) => command.execute_with_dashboard(_ctx).await, + Subcommands::Drop(command) => command.execute().await, + Subcommands::Dump(command) => command.execute().await, + Subcommands::Unwind(command) => command.execute().await, + } + } } diff --git a/bin/reth/src/commands/stage/run.rs b/bin/reth/src/commands/stage/run.rs index 5e2a6552bf54b..b230d4b7691f4 100644 --- a/bin/reth/src/commands/stage/run.rs +++ b/bin/reth/src/commands/stage/run.rs @@ -31,6 +31,9 @@ use reth_stages::{ use std::{any::Any, net::SocketAddr, path::PathBuf, sync::Arc}; use tracing::*; +#[cfg(feature = "open_performance_dashboard")] +use perf_metrics::dashboard::DashboardListener; + /// `reth stage` command #[derive(Debug, Parser)] pub struct Command { @@ -234,6 +237,7 @@ impl Command { StageEnum::StorageHistory => (Box::::default(), None), _ => return Ok(()), }; + if let Some(unwind_stage) = &unwind_stage { assert!(exec_stage.type_id() == unwind_stage.type_id()); } @@ -280,7 +284,24 @@ impl Command { break } } + info!(target: "reth::cli", "reth {} end stage {:?}", SHORT_VERSION, self.stage); Ok(()) } + + #[cfg(feature = "open_performance_dashboard")] + fn set_dashboard(&self, _ctx: crate::runner::CliContext) { + use tokio::sync::mpsc::unbounded_channel; + let (dashboard_tx, dashboard_rx) = unbounded_channel(); + let dashboard_listener = DashboardListener::new(dashboard_rx); + _ctx.task_executor.spawn_critical("dashboard listener task", dashboard_listener); + perf_metrics::set_metric_event_sender(dashboard_tx); + } + + /// Execute `stage` command which support perf test. + #[cfg(feature = "open_performance_dashboard")] + pub async fn execute_with_dashboard(self, _ctx: crate::runner::CliContext) -> eyre::Result<()> { + self.set_dashboard(_ctx); + self.execute().await + } } diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 64a525bfb70f6..ff6a71a34e7ad 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -23,6 +23,9 @@ reth-node-api.workspace = true revm.workspace = true revm-inspectors.workspace = true +# metric +perf-metrics = { workspace = true, optional = true } + # common tracing.workspace = true @@ -39,3 +42,15 @@ optimism = [ "reth-interfaces/optimism", ] js-tracer = ["revm-inspectors/js-tracer"] +enable_opcode_metrics = [ + "revm/enable_opcode_metrics", + "perf-metrics/enable_opcode_metrics", + ] +enable_cache_record = [ + "revm/enable_cache_record", + "reth-provider/enable_cache_record", +] +enable_execution_duration_record = [ + "perf-metrics/enable_execution_duration_record", + "revm/enable_transact_measure", +] \ No newline at end of file diff --git a/crates/revm/src/processor.rs b/crates/revm/src/processor.rs index f10f375d8c3f7..154e557f9e737 100644 --- a/crates/revm/src/processor.rs +++ b/crates/revm/src/processor.rs @@ -313,7 +313,11 @@ where .into()) } let time = Instant::now(); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::start_execute_tx_sub_record(); self.apply_post_execution_state_change(block, total_difficulty)?; + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::apply_post_execution_state_change_record(); self.stats.apply_post_execution_state_changes_duration += time.elapsed(); let time = Instant::now(); @@ -332,6 +336,8 @@ where BundleRetention::PlainState }; self.db_mut().merge_transitions(retention); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::merge_transactions_record(); self.stats.merge_transitions_duration += time.elapsed(); if self.first_block.is_none() { @@ -425,6 +431,8 @@ where block: &BlockWithSenders, total_difficulty: U256, ) -> Result<(), BlockExecutionError> { + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::start_execute_tx_record(); // execute block let receipts = self.execute_inner(block, total_difficulty)?; @@ -443,6 +451,8 @@ where self.stats.receipt_root_duration += time.elapsed(); } + #[cfg(feature = "enable_execution_duration_record")] + let _record = perf_metrics::VerifyAndSaveReceiptsRecord::new(); self.save_receipts(receipts) } @@ -460,6 +470,8 @@ where let mut cumulative_gas_used = 0; let mut receipts = Vec::with_capacity(block.body.len()); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::start_execute_tx_sub_record(); for (sender, transaction) in block.transactions_with_sender() { let time = Instant::now(); // The sum of the transaction’s gas limit, Tg, and the gas utilized in this block prior, @@ -474,6 +486,12 @@ where } // Execute transaction. let ResultAndState { result, state } = self.transact(transaction, *sender)?; + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::transact_record(); + + #[cfg(feature = "enable_opcode_metrics")] + perf_metrics::record_opcode(); + trace!( target: "evm", ?transaction, ?result, ?state, @@ -483,6 +501,8 @@ where let time = Instant::now(); self.db_mut().commit(state); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::commit_changes_record(); self.stats.apply_state_duration += time.elapsed(); @@ -499,6 +519,8 @@ where // convert to reth log logs: result.into_logs().into_iter().map(Into::into).collect(), }); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::add_receipt_record(); } Ok((receipts, cumulative_gas_used)) @@ -520,6 +542,12 @@ where fn size_hint(&self) -> Option { Some(self.evm.context.evm.db.bundle_size_hint()) } + + /// Get state size. + #[cfg(feature = "enable_cache_record")] + fn get_state_size(&self) -> usize { + self.evm.context.evm.db.mem_usage() + } } impl<'a, EvmConfig> PrunableBlockExecutor for EVMProcessor<'a, EvmConfig> diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 3f58fdc837ec2..df4294249be0d 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -55,6 +55,9 @@ rayon.workspace = true num-traits = "0.2.15" auto_impl = "1" +# metric +perf-metrics.workspace = true + [dev-dependencies] # reth reth-primitives = { workspace = true, features = ["test-utils", "arbitrary"] } @@ -82,8 +85,31 @@ criterion = { workspace = true, features = ["async_futures"] } # io serde_json.workspace = true + [features] test-utils = ["reth-interfaces/test-utils", "reth-db/test-utils"] +open_performance_dashboard = [] +enable_opcode_metrics = [ + "perf-metrics/enable_opcode_metrics", + "reth-revm/enable_opcode_metrics", + "open_performance_dashboard", + ] +enable_cache_record = [ + "perf-metrics/enable_cache_record", + "reth-revm/enable_cache_record", + "open_performance_dashboard", + ] +finish_after_execution_stage = [] +enable_execution_duration_record = [ + "perf-metrics/enable_execution_duration_record", + "reth-provider/enable_execution_duration_record", + "reth-revm/enable_execution_duration_record", + "open_performance_dashboard", + ] +enable_tps_gas_record = [ + "perf-metrics/enable_tps_gas_record", + "open_performance_dashboard", + ] [[bench]] name = "criterion" diff --git a/crates/stages/src/pipeline/builder.rs b/crates/stages/src/pipeline/builder.rs index 3e160577fddcb..9644725128643 100644 --- a/crates/stages/src/pipeline/builder.rs +++ b/crates/stages/src/pipeline/builder.rs @@ -41,6 +41,20 @@ where /// [`StageSetBuilder`][crate::StageSetBuilder]. pub fn add_stages>(mut self, set: Set) -> Self { for stage in set.builder().build() { + #[cfg(feature = "finish_after_execution_stage")] + { + match stage.id() { + StageId::MerkleUnwind | + StageId::AccountHashing | + StageId::StorageHashing | + StageId::MerkleExecute | + StageId::TransactionLookup | + StageId::IndexAccountHistory | + StageId::IndexStorageHistory => continue, + _ => println!("Input stage: {:?}", stage.id()), + } + } + self.stages.push(stage); } self diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index beefd98a69eb4..990f82ec756e9 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -117,6 +117,9 @@ impl ExecutionStage { return Ok(ExecOutput::done(input.checkpoint())) } + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::start_record(); + let start_block = input.next_block(); let max_block = input.target(); let prune_modes = self.adjust_prune_modes(provider, start_block, max_block)?; @@ -140,13 +143,19 @@ impl ExecutionStage { let mut cumulative_gas = 0; let batch_start = Instant::now(); + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::record_before_loop(); + for block_number in start_block..=max_block { // Fetch the block let fetch_block_start = Instant::now(); - + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::record_before_td(block_number); let td = provider .header_td_by_number(block_number)? .ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?; + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::record_after_td(); // we need the block's transactions but we don't need the transaction hashes let block = provider @@ -160,6 +169,8 @@ impl ExecutionStage { // Configure the executor to use the current state. trace!(target: "sync::stages::execution", number = block_number, txs = block.body.len(), "Executing block"); + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::record_after_block_with_senders(); // Execute the block let execute_start = Instant::now(); executor.execute_and_verify_receipt(&block, td).map_err(|error| StageError::Block { @@ -168,11 +179,17 @@ impl ExecutionStage { })?; execution_duration += execute_start.elapsed(); - // Gas metrics + // Gas and txs metrics if let Some(metrics_tx) = &mut self.metrics_tx { let _ = metrics_tx.send(MetricEvent::ExecutionStageGas { gas: block.header.gas_used }); } + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::record_after_get_tps( + block_number, + block.body.len() as u64, + block.header.gas_used, + ); stage_progress = block_number; @@ -186,16 +203,26 @@ impl ExecutionStage { cumulative_gas, batch_start.elapsed(), ) { + info!(target: "sync::stages::execution", number = block_number, "Execution batch end"); break } } let time = Instant::now(); let state = executor.take_output_state(); + #[cfg(feature = "open_performance_dashboard")] + perf_metrics::record_after_take_output_state(); + let write_preparation_duration = time.elapsed(); let time = Instant::now(); // write output state.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)?; + + #[cfg(all(feature = "open_performance_dashboard", feature = "enable_cache_record"))] + perf_metrics::record_at_end(executor.get_state_size()); + #[cfg(all(feature = "open_performance_dashboard", not(feature = "enable_cache_record")))] + perf_metrics::record_at_end(0); + let db_write_duration = time.elapsed(); debug!( target: "sync::stages::execution", diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index bc442a5d002f2..b19ebe5982e4a 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -47,6 +47,9 @@ alloy-rlp = { workspace = true, optional = true } # parallel utils rayon.workspace = true +# perf-test +perf-metrics.workspace = true + [dev-dependencies] reth-db = { workspace = true, features = ["test-utils"] } reth-primitives = { workspace = true, features = ["arbitrary", "test-utils"] } @@ -62,3 +65,7 @@ rand.workspace = true [features] test-utils = ["alloy-rlp", "reth-db/test-utils"] optimism = ["reth-primitives/optimism", "reth-interfaces/optimism"] +enable_cache_record = [] +enable_execution_duration_record = [ + "perf-metrics/enable_execution_duration_record" +] \ No newline at end of file diff --git a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs index 6f977c68809a4..d5c43938db407 100644 --- a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs +++ b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs @@ -294,6 +294,8 @@ impl BundleStateWithReceipts { tx: &TX, is_value_known: OriginalValuesKnown, ) -> Result<(), DatabaseError> { + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::start_write_to_db_record(); let (plain_state, reverts) = self.bundle.into_plain_state_and_reverts(is_value_known); StateReverts(reverts).write_to_db(tx, self.first_block)?; @@ -314,11 +316,22 @@ impl BundleStateWithReceipts { let first_tx_index = body_indices.first_tx_num(); for (tx_idx, receipt) in receipts.into_iter().enumerate() { if let Some(receipt) = receipt { + #[cfg(feature = "enable_execution_duration_record")] + let _record = perf_metrics::ReceiptsWrite::new( + std::mem::size_of::() + + receipt + .logs + .iter() + .map(|log| log.topics.len() * 32 + log.data.0.len()) + .sum::(), + ); receipts_cursor.append(first_tx_index + tx_idx as u64, receipt)?; } } } } + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_write_receipts_time(); StateChanges(plain_state).write_to_db(tx)?; diff --git a/crates/storage/provider/src/bundle_state/state_changes.rs b/crates/storage/provider/src/bundle_state/state_changes.rs index a62606dedebc3..52ef03e9eaaa6 100644 --- a/crates/storage/provider/src/bundle_state/state_changes.rs +++ b/crates/storage/provider/src/bundle_state/state_changes.rs @@ -26,6 +26,8 @@ impl StateChanges { self.0.accounts.par_sort_by_key(|a| a.0); self.0.storage.par_sort_by_key(|a| a.address); self.0.contracts.par_sort_by_key(|a| a.0); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_sort_time(); // Write new account state tracing::trace!(target: "provider::bundle_state", len = self.0.accounts.len(), "Writing new account state"); @@ -34,19 +36,30 @@ impl StateChanges { for (address, account) in self.0.accounts.into_iter() { if let Some(account) = account { tracing::trace!(target: "provider::bundle_state", ?address, "Updating plain state account"); + #[cfg(feature = "enable_execution_duration_record")] + // sizeof(Address) + sizeof(Account) = 100 + let _record = perf_metrics::StateAccountWrite::new(100usize); accounts_cursor.upsert(address, into_reth_acc(account))?; } else if accounts_cursor.seek_exact(address)?.is_some() { tracing::trace!(target: "provider::bundle_state", ?address, "Deleting plain state account"); accounts_cursor.delete_current()?; } } + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_state_account_time(); // Write bytecode tracing::trace!(target: "provider::bundle_state", len = self.0.contracts.len(), "Writing bytecodes"); let mut bytecodes_cursor = tx.cursor_write::()?; for (hash, bytecode) in self.0.contracts.into_iter() { + #[cfg(feature = "enable_execution_duration_record")] + // size_of_val(hash) + size_of::() = 88 + let _record = + perf_metrics::StateBytecodeWrite::new(88usize + bytecode.bytecode.0.len()); bytecodes_cursor.upsert(hash, Bytecode(bytecode))?; } + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_state_bytecode_time(); // Write new storage state and wipe storage if needed. tracing::trace!(target: "provider::bundle_state", len = self.0.storage.len(), "Writing new storage state"); @@ -73,10 +86,15 @@ impl StateChanges { } if entry.value != U256::ZERO { + #[cfg(feature = "enable_execution_duration_record")] + // sizeof(Address) + sizeof(StorageEntry) = 84 + let _record = perf_metrics::StateStorageWrite::new(84usize); storages_cursor.upsert(address, entry)?; } } } + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_state_storage_time(); Ok(()) } } diff --git a/crates/storage/provider/src/bundle_state/state_reverts.rs b/crates/storage/provider/src/bundle_state/state_reverts.rs index 8b7d5c7c283e4..8fe57fe0b9e1c 100644 --- a/crates/storage/provider/src/bundle_state/state_reverts.rs +++ b/crates/storage/provider/src/bundle_state/state_reverts.rs @@ -31,6 +31,9 @@ impl StateReverts { ) -> Result<(), DatabaseError> { // Write storage changes tracing::trace!(target: "provider::reverts", "Writing storage changes"); + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::start_write_to_db_sub_record(); + let mut storages_cursor = tx.cursor_dup_write::()?; let mut storage_changeset_cursor = tx.cursor_dup_write::()?; for (block_index, mut storage_changes) in self.0.storage.into_iter().enumerate() { @@ -66,10 +69,15 @@ impl StateReverts { tracing::trace!(target: "provider::reverts", ?address, ?storage, "Writing storage reverts"); for (key, value) in StorageRevertsIter::new(storage, wiped_storage) { + #[cfg(feature = "enable_execution_duration_record")] + // sizeof(B256) + sizeof(StorageEntry) = 96 + let _record = perf_metrics::RevertsStorageWrite::new(96usize); storage_changeset_cursor.append_dup(storage_id, StorageEntry { key, value })?; } } } + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_revert_storage_time(); // Write account changes tracing::trace!(target: "provider::reverts", "Writing account changes"); @@ -79,12 +87,17 @@ impl StateReverts { // Sort accounts by address. account_block_reverts.par_sort_by_key(|a| a.0); for (address, info) in account_block_reverts { + #[cfg(feature = "enable_execution_duration_record")] + // sizeof(Address) + sizeof(AccountBeforeTx) = 124 + let _record = perf_metrics::RevertsAccountWrite::new(124usize); account_changeset_cursor.append_dup( block_number, AccountBeforeTx { address, info: info.map(into_reth_acc) }, )?; } } + #[cfg(feature = "enable_execution_duration_record")] + perf_metrics::record_revert_account_time(); Ok(()) } diff --git a/crates/storage/provider/src/traits/executor.rs b/crates/storage/provider/src/traits/executor.rs index bcb95f83bfdbf..c574dc1754448 100644 --- a/crates/storage/provider/src/traits/executor.rs +++ b/crates/storage/provider/src/traits/executor.rs @@ -61,6 +61,12 @@ pub trait BlockExecutor { /// Returns the size hint of current in-memory changes. fn size_hint(&self) -> Option; + + /// Get CacheDb metric size. + #[cfg(feature = "enable_cache_record")] + fn get_state_size(&self) -> usize { + 0 + } } /// A [BlockExecutor] capable of in-memory pruning of the data that will be written to the database.