diff --git a/Cargo.lock b/Cargo.lock index fd054989..15246503 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7841,7 +7841,6 @@ dependencies = [ "derive_more 1.0.0", "eyre", "rbuilder", - "reth-db-api", "reth-primitives", "reth-provider", "tokio", diff --git a/crates/rbuilder/src/backtest/execute.rs b/crates/rbuilder/src/backtest/execute.rs index 053d8bb5..4c392cdb 100644 --- a/crates/rbuilder/src/backtest/execute.rs +++ b/crates/rbuilder/src/backtest/execute.rs @@ -7,14 +7,13 @@ use crate::{ }, live_builder::cli::LiveBuilderConfig, primitives::{OrderId, SimulatedOrder}, + provider::StateProviderFactory, utils::{clean_extradata, Signer}, }; use ahash::HashSet; use alloy_primitives::{Address, U256}; use reth::revm::cached::CachedReads; use reth_chainspec::ChainSpec; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory}; use serde::{Deserialize, Serialize}; use std::{cell::RefCell, rc::Rc, sync::Arc}; @@ -88,6 +87,7 @@ where builder_signer.address, block_data.winning_bid_trace.proposer_fee_recipient, Some(builder_signer), + Arc::from(provider.root_hasher(block_data.winning_bid_trace.parent_hash)), ); let (sim_orders, sim_errors) = simulate_all_orders_with_sim_tree(provider.clone(), &ctx, &orders, false)?; @@ -107,7 +107,7 @@ where } #[allow(clippy::too_many_arguments)] -pub fn backtest_simulate_block( +pub fn backtest_simulate_block( block_data: BlockData, provider: P, chain_spec: Arc, @@ -118,12 +118,7 @@ pub fn backtest_simulate_block( sbundle_mergeabe_signers: &[Address], ) -> eyre::Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { let BacktestBlockInput { diff --git a/crates/rbuilder/src/backtest/redistribute/cli/mod.rs b/crates/rbuilder/src/backtest/redistribute/cli/mod.rs index 5c72b5c8..215b7b78 100644 --- a/crates/rbuilder/src/backtest/redistribute/cli/mod.rs +++ b/crates/rbuilder/src/backtest/redistribute/cli/mod.rs @@ -6,12 +6,11 @@ use crate::{ BlockData, HistoricalDataStorage, }, live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig}, + provider::StateProviderFactory, }; use alloy_primitives::utils::format_ether; use clap::Parser; use csv_output::{CSVOutputRow, CSVResultWriter}; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory}; use std::{io, path::PathBuf}; use tracing::info; @@ -107,7 +106,7 @@ where Ok(()) } -fn process_redisribution( +fn process_redisribution( block_data: BlockData, csv_writer: Option<&mut CSVResultWriter>, json_accum: Option<&mut Vec>, @@ -116,12 +115,7 @@ fn process_redisribution( distribute_to_mempool_txs: bool, ) -> eyre::Result<()> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { let block_number = block_data.block_number; diff --git a/crates/rbuilder/src/backtest/redistribute/mod.rs b/crates/rbuilder/src/backtest/redistribute/mod.rs index 1d26a940..b6a1748c 100644 --- a/crates/rbuilder/src/backtest/redistribute/mod.rs +++ b/crates/rbuilder/src/backtest/redistribute/mod.rs @@ -16,6 +16,7 @@ use crate::{ }, live_builder::cli::LiveBuilderConfig, primitives::{Order, OrderId}, + provider::StateProviderFactory, utils::{signed_uint_delta, u256decimal_serde_helper}, }; use ahash::{HashMap, HashSet}; @@ -23,8 +24,6 @@ use alloy_primitives::{utils::format_ether, Address, B256, I256, U256}; pub use cli::run_backtest_redistribute; use rayon::prelude::*; use reth_chainspec::ChainSpec; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory}; use serde::{Deserialize, Serialize}; use std::{ cmp::{max, min}, @@ -117,19 +116,14 @@ pub struct RedistributionBlockOutput { pub joint_contribution: Vec, } -pub fn calc_redistributions( +pub fn calc_redistributions( provider: P, config: &ConfigType, block_data: BlockData, distribute_to_mempool_txs: bool, ) -> eyre::Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { let _block_span = info_span!("block", block = block_data.block_number).entered(); @@ -280,7 +274,7 @@ fn restore_available_landed_orders

( included_orders_available: &[OrdersWithTimestamp], ) -> eyre::Result> where - P: StateProviderFactory + HeaderProvider + Clone + 'static, + P: StateProviderFactory + Clone + 'static, { let block_txs = sim_historical_block( provider.clone(), @@ -480,18 +474,13 @@ impl ResultsWithoutExclusion { } } -fn calculate_backtest_without_exclusion( +fn calculate_backtest_without_exclusion( provider: P, config: &ConfigType, block_data: BlockData, ) -> eyre::Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { let ExclusionResult { @@ -548,7 +537,7 @@ impl ExclusionResults { } } -fn calculate_backtest_identity_and_order_exclusion( +fn calculate_backtest_identity_and_order_exclusion( provider: P, config: &ConfigType, block_data: BlockData, @@ -556,12 +545,7 @@ fn calculate_backtest_identity_and_order_exclusion( results_without_exclusion: &ResultsWithoutExclusion, ) -> eyre::Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { let included_orders_exclusion = { @@ -621,7 +605,7 @@ where }) } -fn calc_joint_exclusion_results( +fn calc_joint_exclusion_results( provider: P, config: &ConfigType, block_data: BlockData, @@ -631,12 +615,7 @@ fn calc_joint_exclusion_results( distribute_to_mempool_txs: bool, ) -> eyre::Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { // calculate identities that are possibly connected @@ -962,19 +941,14 @@ struct ExclusionResult { } /// calculate block profit excluding some orders -fn calc_profit_after_exclusion( +fn calc_profit_after_exclusion( provider: P, config: &ConfigType, block_data: &BlockData, exclusion_input: ExclusionInput, ) -> eyre::Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, ConfigType: LiveBuilderConfig, { let block_data_with_excluded = { diff --git a/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs b/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs index 5f570f9b..b4392434 100644 --- a/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs +++ b/crates/rbuilder/src/backtest/restore_landed_orders/resim_landed_block.rs @@ -3,6 +3,7 @@ use crate::{ evm_inspector::SlotKey, tracers::AccumulatorSimulationTracer, BlockBuildingContext, BlockState, PartialBlock, PartialBlockFork, }, + provider::StateProviderFactory, utils::{extract_onchain_block_txs, find_suggested_fee_recipient, signed_uint_delta}, }; use ahash::{HashMap, HashSet}; @@ -10,7 +11,6 @@ use alloy_primitives::{TxHash, B256, I256}; use eyre::Context; use reth_chainspec::ChainSpec; use reth_primitives::{Receipt, TransactionSignedEcRecovered}; -use reth_provider::StateProviderFactory; use std::sync::Arc; #[derive(Debug)] @@ -40,7 +40,9 @@ where let txs = extract_onchain_block_txs(&onchain_block)?; let suggested_fee_recipient = find_suggested_fee_recipient(&onchain_block, &txs); + let coinbase = onchain_block.header.beneficiary; + let parent_hash = onchain_block.header.parent_hash; let ctx = BlockBuildingContext::from_onchain_block( onchain_block, @@ -50,6 +52,7 @@ where coinbase, suggested_fee_recipient, None, + Arc::from(provider.root_hasher(parent_hash)), ); let state_provider = provider.history_by_block_hash(ctx.attributes.parent)?; diff --git a/crates/rbuilder/src/bin/debug-bench-machine.rs b/crates/rbuilder/src/bin/debug-bench-machine.rs index 417f6648..2f7c1f1e 100644 --- a/crates/rbuilder/src/bin/debug-bench-machine.rs +++ b/crates/rbuilder/src/bin/debug-bench-machine.rs @@ -9,9 +9,10 @@ use itertools::Itertools; use rbuilder::{ building::{BlockBuildingContext, BlockState, PartialBlock, PartialBlockFork}, live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig, config::Config}, + provider::StateProviderFactory, utils::{extract_onchain_block_txs, find_suggested_fee_recipient, http_provider}, }; -use reth::{providers::BlockNumReader, revm::cached::CachedReads}; +use reth::revm::cached::CachedReads; use reth_provider::StateProvider; use std::{path::PathBuf, sync::Arc, time::Instant}; use tracing::{debug, info}; @@ -61,6 +62,7 @@ async fn main() -> eyre::Result<()> { let coinbase = onchain_block.header.beneficiary; + let parent_hash = onchain_block.header.parent_hash; let ctx = BlockBuildingContext::from_onchain_block( onchain_block, chain_spec, @@ -69,6 +71,7 @@ async fn main() -> eyre::Result<()> { coinbase, suggested_fee_recipient, None, + Arc::from(provider_factory.root_hasher(parent_hash)), ); let state_provider = Arc::::from( @@ -84,9 +87,6 @@ async fn main() -> eyre::Result<()> { let ctx = ctx.clone(); let txs = txs.clone(); let state_provider = state_provider.clone(); - let factory = provider_factory.clone(); - let config = config.clone(); - let root_hash_config = config.base_config.live_root_hash_config()?; let (new_cached_reads, build_time, finalize_time) = tokio::task::spawn_blocking(move || -> eyre::Result<_> { let partial_block = PartialBlock::new(true, None); @@ -112,12 +112,7 @@ async fn main() -> eyre::Result<()> { let build_time = build_time.elapsed(); let finalize_time = Instant::now(); - let finalized_block = partial_block.finalize( - &mut state, - &ctx, - factory.clone(), - root_hash_config.clone(), - )?; + let finalized_block = partial_block.finalize(&mut state, &ctx)?; let finalize_time = finalize_time.elapsed(); debug!( diff --git a/crates/rbuilder/src/bin/dummy-builder.rs b/crates/rbuilder/src/bin/dummy-builder.rs index b4d9e688..057558fc 100644 --- a/crates/rbuilder/src/bin/dummy-builder.rs +++ b/crates/rbuilder/src/bin/dummy-builder.rs @@ -34,14 +34,13 @@ use rbuilder::{ mev_boost::{MevBoostRelay, RelayConfig}, SimulatedOrder, }, - roothash::RootHashConfig, + provider::StateProviderFactory, utils::{ProviderFactoryReopener, Signer}, }; use reth_chainspec::MAINNET; -use reth_db::{database::Database, DatabaseEnv}; +use reth_db::DatabaseEnv; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use tokio::{ signal::ctrl_c, sync::{broadcast, mpsc}, @@ -89,7 +88,6 @@ async fn main() -> eyre::Result<()> { mpsc::channel(order_input_config.input_channel_buffer_size); let builder = LiveBuilder::< ProviderFactoryReopener>>, - Arc, MevBoostSlotDataGenerator, > { watchdog_timeout: Some(Duration::from_secs(10000)), @@ -103,6 +101,7 @@ async fn main() -> eyre::Result<()> { None, None, chain_spec.clone(), + None, )?, coinbase_signer: Signer::random(), extra_data: Vec::new(), @@ -199,22 +198,17 @@ impl DummyBuildingAlgorithm { } } - fn build_block( + fn build_block

( &self, orders: Vec, provider: P, ctx: &BlockBuildingContext, ) -> eyre::Result> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let mut block_building_helper = BlockBuildingHelperFromProvider::new( provider.clone(), - RootHashConfig::live_config(false, false), ctx.clone(), None, BUILDER_NAME.to_string(), @@ -231,13 +225,9 @@ impl DummyBuildingAlgorithm { } } -impl BlockBuildingAlgorithm for DummyBuildingAlgorithm +impl

BlockBuildingAlgorithm

for DummyBuildingAlgorithm where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { fn name(&self) -> String { BUILDER_NAME.to_string() diff --git a/crates/rbuilder/src/building/builders/block_building_helper.rs b/crates/rbuilder/src/building/builders/block_building_helper.rs index 0bee97bf..d9acc2cd 100644 --- a/crates/rbuilder/src/building/builders/block_building_helper.rs +++ b/crates/rbuilder/src/building/builders/block_building_helper.rs @@ -1,13 +1,9 @@ +use alloy_primitives::{utils::format_ether, U256}; +use reth::revm::cached::CachedReads; use std::{ cmp::max, - marker::PhantomData, time::{Duration, Instant}, }; - -use alloy_primitives::{utils::format_ether, U256}; -use reth::revm::cached::CachedReads; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use time::OffsetDateTime; use tokio_util::sync::CancellationToken; use tracing::{debug, error, trace}; @@ -20,7 +16,7 @@ use crate::{ PartialBlock, Sorting, }, primitives::SimulatedOrder, - roothash::RootHashConfig, + provider::StateProviderFactory, telemetry, utils::{check_block_hash_reader_health, HistoricalBlockError}, }; @@ -85,13 +81,9 @@ pub trait BlockBuildingHelper: Send + Sync { /// Implementation of BlockBuildingHelper based on a generic Provider #[derive(Clone)] -pub struct BlockBuildingHelperFromProvider +pub struct BlockBuildingHelperFromProvider

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory, { /// Balance of fee recipient before we stared building. _fee_recipient_balance_start: U256, @@ -108,10 +100,8 @@ where built_block_trace: BuiltBlockTrace, /// Needed to get the initial state and the final root hash calculation. provider: P, - root_hash_config: RootHashConfig, /// Token to cancel in case of fatal error (if we believe that it's impossible to build for this block). cancel_on_fatal_error: CancellationToken, - phantom: PhantomData, } #[derive(Debug, thiserror::Error)] @@ -155,13 +145,9 @@ pub struct FinalizeBlockResult { pub cached_reads: CachedReads, } -impl BlockBuildingHelperFromProvider +impl

BlockBuildingHelperFromProvider

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { /// allow_tx_skip: see [`PartialBlockFork`] /// Performs initialization: @@ -171,7 +157,6 @@ where #[allow(clippy::too_many_arguments)] pub fn new( provider: P, - root_hash_config: RootHashConfig, building_ctx: BlockBuildingContext, cached_reads: Option, builder_name: String, @@ -217,9 +202,7 @@ where building_ctx, built_block_trace: BuiltBlockTrace::new(), provider, - root_hash_config, cancel_on_fatal_error, - phantom: PhantomData, }) } @@ -300,13 +283,9 @@ where } } -impl BlockBuildingHelper for BlockBuildingHelperFromProvider +impl

BlockBuildingHelper for BlockBuildingHelperFromProvider

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { /// Forwards to partial_block and updates trace. fn commit_order( @@ -370,12 +349,10 @@ where let sim_gas_used = self.partial_block.tracer.used_gas; let block_number = self.building_context().block(); - let finalized_block = match self.partial_block.finalize( - &mut self.block_state, - &self.building_ctx, - self.provider.clone(), - self.root_hash_config, - ) { + let finalized_block = match self + .partial_block + .finalize(&mut self.block_state, &self.building_ctx) + { Ok(finalized_block) => finalized_block, Err(err) => { if err.is_consistent_db_view_err() { diff --git a/crates/rbuilder/src/building/builders/mock_block_building_helper.rs b/crates/rbuilder/src/building/builders/mock_block_building_helper.rs index 26d38112..6f32f055 100644 --- a/crates/rbuilder/src/building/builders/mock_block_building_helper.rs +++ b/crates/rbuilder/src/building/builders/mock_block_building_helper.rs @@ -1,3 +1,6 @@ +use crate::live_builder::simulation::SimulatedOrderCommand; +use crate::provider::RootHasher; +use crate::roothash::RootHashError; use crate::{ building::{ BlockBuildingContext, BuiltBlockTrace, CriticalCommitOrderError, ExecutionError, @@ -5,10 +8,14 @@ use crate::{ }, primitives::SimulatedOrder, }; +use alloy_primitives::B256; use alloy_primitives::U256; +use reth::providers::ExecutionOutcome; use reth::revm::cached::CachedReads; use reth_primitives::SealedBlock; use time::OffsetDateTime; +use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; use super::{ block_building_helper::{BlockBuildingHelper, BlockBuildingHelperError, FinalizeBlockResult}, @@ -110,3 +117,19 @@ impl BlockBuildingHelper for MockBlockBuildingHelper { "Mock" } } + +#[derive(Debug)] +pub struct MockRootHasher {} + +impl RootHasher for MockRootHasher { + fn run_prefetcher( + &self, + _simulated_orders: broadcast::Receiver, + _cancel: CancellationToken, + ) { + } + + fn state_root(&self, _outcome: &ExecutionOutcome) -> Result { + Ok(B256::default()) + } +} diff --git a/crates/rbuilder/src/building/builders/mod.rs b/crates/rbuilder/src/building/builders/mod.rs index 96d5cc84..2c6bba23 100644 --- a/crates/rbuilder/src/building/builders/mod.rs +++ b/crates/rbuilder/src/building/builders/mod.rs @@ -8,7 +8,7 @@ use crate::{ building::{BlockBuildingContext, BuiltBlockTrace, SimulatedOrderSink, Sorting}, live_builder::{payload_events::MevBoostSlotData, simulation::SimulatedOrderCommand}, primitives::{AccountNonce, OrderId, SimulatedOrder}, - roothash::RootHashConfig, + provider::StateProviderFactory, utils::{is_provider_factory_health_error, NonceCache}, }; use ahash::HashSet; @@ -16,10 +16,8 @@ use alloy_eips::eip4844::BlobTransactionSidecar; use alloy_primitives::{Address, Bytes, B256}; use block_building_helper::BlockBuildingHelper; use reth::{primitives::SealedBlock, revm::cached::CachedReads}; -use reth_db::Database; use reth_errors::ProviderError; -use reth_provider::{DatabaseProviderFactory, StateProviderFactory}; -use std::{fmt::Debug, marker::PhantomData, sync::Arc}; +use std::{fmt::Debug, sync::Arc}; use tokio::sync::{broadcast, broadcast::error::TryRecvError}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; @@ -39,15 +37,13 @@ pub struct Block { } #[derive(Debug)] -pub struct LiveBuilderInput { +pub struct LiveBuilderInput

{ pub provider: P, - pub root_hash_config: RootHashConfig, pub ctx: BlockBuildingContext, pub input: broadcast::Receiver, pub sink: Arc, pub builder_name: String, pub cancel: CancellationToken, - phantom: PhantomData, } /// Struct that helps reading new orders/cancelations @@ -214,10 +210,9 @@ pub struct BlockBuildingAlgorithmInput

{ /// Algorithm to build blocks /// build_blocks should send block to input.sink until input.cancel is cancelled. /// slot_bidder should be used to decide how much to bid. -pub trait BlockBuildingAlgorithm: Debug + Send + Sync +pub trait BlockBuildingAlgorithm

: Debug + Send + Sync where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory + StateProviderFactory + Clone + 'static, + P: StateProviderFactory, { fn name(&self) -> String; fn build_blocks(&self, input: BlockBuildingAlgorithmInput

); diff --git a/crates/rbuilder/src/building/builders/ordering_builder.rs b/crates/rbuilder/src/building/builders/ordering_builder.rs index 9d4da9f0..ba5313f8 100644 --- a/crates/rbuilder/src/building/builders/ordering_builder.rs +++ b/crates/rbuilder/src/building/builders/ordering_builder.rs @@ -14,17 +14,12 @@ use crate::{ BlockBuildingContext, ExecutionError, PrioritizedOrderStore, SimulatedOrderSink, Sorting, }, primitives::{AccountNonce, OrderId}, - roothash::RootHashConfig, + provider::StateProviderFactory, }; use ahash::{HashMap, HashSet}; use reth::revm::cached::CachedReads; -use reth_db::database::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use serde::Deserialize; -use std::{ - marker::PhantomData, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{error, info_span, trace}; @@ -61,13 +56,9 @@ impl OrderingBuilderConfig { } } -pub fn run_ordering_builder(input: LiveBuilderInput, config: &OrderingBuilderConfig) +pub fn run_ordering_builder

(input: LiveBuilderInput

, config: &OrderingBuilderConfig) where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let mut order_intake_consumer = OrderIntakeConsumer::new( input.provider.clone(), @@ -81,7 +72,6 @@ where input.builder_name, input.ctx, config.clone(), - input.root_hash_config, ); // this is a hack to mark used orders until built block trace is implemented as a sane thing @@ -130,16 +120,12 @@ where } } -pub fn backtest_simulate_block( +pub fn backtest_simulate_block

( ordering_config: OrderingBuilderConfig, input: BacktestSimulateBlockInput<'_, P>, ) -> eyre::Result<(Block, CachedReads)> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let use_suggested_fee_recipient_as_coinbase = ordering_config.coinbase_payment; let state_provider = input @@ -152,7 +138,6 @@ where input.builder_name, input.ctx.clone(), ordering_config, - RootHashConfig::skip_root_hash(), ) .with_cached_reads(input.cached_reads.unwrap_or_default()); let block_builder = builder.build_block( @@ -174,12 +159,11 @@ where } #[derive(Debug)] -pub struct OrderingBuilderContext { +pub struct OrderingBuilderContext

{ provider: P, builder_name: String, ctx: BlockBuildingContext, config: OrderingBuilderConfig, - root_hash_config: RootHashConfig, // caches cached_reads: Option, @@ -187,35 +171,26 @@ pub struct OrderingBuilderContext { // scratchpad failed_orders: HashSet, order_attempts: HashMap, - - phantom: PhantomData, } -impl OrderingBuilderContext +impl

OrderingBuilderContext

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { pub fn new( provider: P, builder_name: String, ctx: BlockBuildingContext, config: OrderingBuilderConfig, - root_hash_config: RootHashConfig, ) -> Self { Self { provider, builder_name, ctx, config, - root_hash_config, cached_reads: None, failed_orders: HashSet::default(), order_attempts: HashMap::default(), - phantom: PhantomData, } } @@ -255,7 +230,6 @@ where let mut block_building_helper = BlockBuildingHelperFromProvider::new( self.provider.clone(), - self.root_hash_config.clone(), new_ctx, self.cached_reads.take(), self.builder_name.clone(), @@ -339,32 +313,19 @@ where #[derive(Debug)] pub struct OrderingBuildingAlgorithm { - root_hash_config: RootHashConfig, config: OrderingBuilderConfig, name: String, } impl OrderingBuildingAlgorithm { - pub fn new( - root_hash_config: RootHashConfig, - config: OrderingBuilderConfig, - name: String, - ) -> Self { - Self { - root_hash_config, - config, - name, - } + pub fn new(config: OrderingBuilderConfig, name: String) -> Self { + Self { config, name } } } -impl BlockBuildingAlgorithm for OrderingBuildingAlgorithm +impl

BlockBuildingAlgorithm

for OrderingBuildingAlgorithm where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { fn name(&self) -> String { self.name.clone() @@ -373,13 +334,11 @@ where fn build_blocks(&self, input: BlockBuildingAlgorithmInput

) { let live_input = LiveBuilderInput { provider: input.provider, - root_hash_config: self.root_hash_config.clone(), ctx: input.ctx.clone(), input: input.input, sink: input.sink, builder_name: self.name.clone(), cancel: input.cancel, - phantom: Default::default(), }; run_ordering_builder(live_input, &self.config); } diff --git a/crates/rbuilder/src/building/builders/parallel_builder/block_building_result_assembler.rs b/crates/rbuilder/src/building/builders/parallel_builder/block_building_result_assembler.rs index 7e1c02bc..9fde7c2c 100644 --- a/crates/rbuilder/src/building/builders/parallel_builder/block_building_result_assembler.rs +++ b/crates/rbuilder/src/building/builders/parallel_builder/block_building_result_assembler.rs @@ -5,9 +5,7 @@ use super::{ use ahash::HashMap; use alloy_primitives::utils::format_ether; use reth::revm::cached::CachedReads; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; -use std::{marker::PhantomData, sync::Arc, time::Instant}; +use std::{sync::Arc, time::Instant}; use time::OffsetDateTime; use tokio_util::sync::CancellationToken; use tracing::{info_span, trace}; @@ -20,11 +18,11 @@ use crate::{ }, BlockBuildingContext, }, - roothash::RootHashConfig, + provider::StateProviderFactory, }; /// Assembles block building results from the best orderings of order groups. -pub struct BlockBuildingResultAssembler { +pub struct BlockBuildingResultAssembler

{ provider: P, ctx: BlockBuildingContext, cancellation_token: CancellationToken, @@ -32,22 +30,16 @@ pub struct BlockBuildingResultAssembler { discard_txs: bool, coinbase_payment: bool, can_use_suggested_fee_recipient_as_coinbase: bool, - root_hash_config: RootHashConfig, builder_name: String, sink: Option>, best_results: Arc, run_id: u64, last_version: Option, - phantom: PhantomData, } -impl BlockBuildingResultAssembler +impl

BlockBuildingResultAssembler

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { /// Creates a new `BlockBuildingResultAssembler`. /// @@ -60,7 +52,6 @@ where #[allow(clippy::too_many_arguments)] pub fn new( config: &ParallelBuilderConfig, - root_hash_config: RootHashConfig, best_results: Arc, provider: P, ctx: BlockBuildingContext, @@ -77,13 +68,11 @@ where discard_txs: config.discard_txs, coinbase_payment: config.coinbase_payment, can_use_suggested_fee_recipient_as_coinbase, - root_hash_config, builder_name, sink, best_results, run_id: 0, last_version: None, - phantom: PhantomData, } } @@ -205,7 +194,6 @@ where let mut block_building_helper = BlockBuildingHelperFromProvider::new( self.provider.clone(), - self.root_hash_config.clone(), ctx, self.cached_reads.clone(), self.builder_name.clone(), @@ -273,7 +261,6 @@ where ) -> eyre::Result> { let mut block_building_helper = BlockBuildingHelperFromProvider::new( self.provider.clone(), - self.root_hash_config.clone(), // Adjust as needed for backtest self.ctx.clone(), None, // No cached reads for backtest start String::from("backtest_builder"), diff --git a/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolvers.rs b/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolvers.rs index 0ffebc5f..390095af 100644 --- a/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolvers.rs +++ b/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolvers.rs @@ -4,7 +4,6 @@ use eyre::Result; use itertools::Itertools; use rand::{seq::SliceRandom, SeedableRng}; use reth::{providers::StateProvider, revm::cached::CachedReads}; -use reth_provider::StateProviderFactory; use std::sync::Arc; use tokio_util::sync::CancellationToken; use tracing::trace; @@ -17,6 +16,7 @@ use super::{ use crate::{ building::{BlockBuildingContext, BlockState, ExecutionError, ExecutionResult, PartialBlock}, primitives::{OrderId, SimulatedOrder}, + provider::StateProviderFactory, }; /// Context for resolving conflicts in merging tasks. @@ -31,7 +31,7 @@ pub struct ResolverContext

{ impl

ResolverContext

where - P: StateProviderFactory + Clone + 'static, + P: StateProviderFactory, { /// Creates a new `ResolverContext`. /// diff --git a/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolving_pool.rs b/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolving_pool.rs index f766c89a..741a8e76 100644 --- a/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolving_pool.rs +++ b/crates/rbuilder/src/building/builders/parallel_builder/conflict_resolving_pool.rs @@ -1,7 +1,6 @@ use alloy_primitives::utils::format_ether; use crossbeam_queue::SegQueue; use eyre::Result; -use reth_provider::StateProviderFactory; use std::{ sync::{mpsc as std_mpsc, Arc}, thread, @@ -16,6 +15,7 @@ use super::{ ConflictTask, GroupId, ResolutionResult, TaskPriority, }; use crate::building::BlockBuildingContext; +use crate::provider::StateProviderFactory; pub type TaskQueue = Arc>; diff --git a/crates/rbuilder/src/building/builders/parallel_builder/mod.rs b/crates/rbuilder/src/building/builders/parallel_builder/mod.rs index 51f6bad7..5cccb287 100644 --- a/crates/rbuilder/src/building/builders/parallel_builder/mod.rs +++ b/crates/rbuilder/src/building/builders/parallel_builder/mod.rs @@ -33,11 +33,9 @@ use crate::{ BacktestSimulateBlockInput, Block, BlockBuildingAlgorithm, BlockBuildingAlgorithmInput, LiveBuilderInput, }, - roothash::RootHashConfig, + provider::StateProviderFactory, }; use reth::revm::cached::CachedReads; -use reth_db::database::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use self::{ block_building_result_assembler::BlockBuildingResultAssembler, @@ -72,26 +70,22 @@ fn get_shared_data_structures() -> (Arc, TaskQueue) { (best_results, task_queue) } -struct ParallelBuilder { +struct ParallelBuilder

{ order_intake_consumer: OrderIntakeStore, conflict_finder: ConflictFinder, conflict_task_generator: ConflictTaskGenerator, conflict_resolving_pool: ConflictResolvingPool

, results_aggregator: ResultsAggregator, - block_building_result_assembler: BlockBuildingResultAssembler, + block_building_result_assembler: BlockBuildingResultAssembler

, } -impl ParallelBuilder +impl

ParallelBuilder

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { /// Creates a ParallelBuilder. /// Sets up the various components and communication channels. - pub fn new(input: LiveBuilderInput, config: &ParallelBuilderConfig) -> Self { + pub fn new(input: LiveBuilderInput

, config: &ParallelBuilderConfig) -> Self { let (group_result_sender, group_result_receiver) = get_communication_channels(); let group_result_sender_for_task_generator = group_result_sender.clone(); @@ -121,7 +115,6 @@ where let block_building_result_assembler = BlockBuildingResultAssembler::new( config, - input.root_hash_config, Arc::clone(&best_results), input.provider.clone(), input.ctx.clone(), @@ -181,13 +174,9 @@ where /// /// # Type Parameters /// * `DB`: The database type, which must implement Database, Clone, and have a static lifetime. -pub fn run_parallel_builder(input: LiveBuilderInput, config: &ParallelBuilderConfig) +pub fn run_parallel_builder

(input: LiveBuilderInput

, config: &ParallelBuilderConfig) where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let cancel_for_results_aggregator = input.cancel.clone(); let cancel_for_block_building_result_assembler = input.cancel.clone(); @@ -268,16 +257,12 @@ fn run_order_intake( } } -pub fn parallel_build_backtest( +pub fn parallel_build_backtest

( input: BacktestSimulateBlockInput<'_, P>, config: ParallelBuilderConfig, ) -> Result<(Block, CachedReads)> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let start_time = Instant::now(); @@ -329,7 +314,6 @@ where let assembler_start = Instant::now(); let block_building_result_assembler = BlockBuildingResultAssembler::new( &config, - RootHashConfig::skip_root_hash(), Arc::clone(&best_results), input.provider.clone(), input.ctx.clone(), @@ -381,32 +365,19 @@ where #[derive(Debug)] pub struct ParallelBuildingAlgorithm { - root_hash_config: RootHashConfig, config: ParallelBuilderConfig, name: String, } impl ParallelBuildingAlgorithm { - pub fn new( - root_hash_config: RootHashConfig, - config: ParallelBuilderConfig, - name: String, - ) -> Self { - Self { - root_hash_config, - config, - name, - } + pub fn new(config: ParallelBuilderConfig, name: String) -> Self { + Self { config, name } } } -impl BlockBuildingAlgorithm for ParallelBuildingAlgorithm +impl

BlockBuildingAlgorithm

for ParallelBuildingAlgorithm where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { fn name(&self) -> String { self.name.clone() @@ -415,13 +386,11 @@ where fn build_blocks(&self, input: BlockBuildingAlgorithmInput

) { let live_input = LiveBuilderInput { provider: input.provider, - root_hash_config: self.root_hash_config.clone(), ctx: input.ctx.clone(), input: input.input, sink: input.sink, builder_name: self.name.clone(), cancel: input.cancel, - phantom: Default::default(), }; run_parallel_builder(live_input, &self.config); } diff --git a/crates/rbuilder/src/building/mod.rs b/crates/rbuilder/src/building/mod.rs index 7f59bf0e..72abc528 100644 --- a/crates/rbuilder/src/building/mod.rs +++ b/crates/rbuilder/src/building/mod.rs @@ -12,14 +12,13 @@ pub mod testing; pub mod tracers; use alloy_consensus::{Header, EMPTY_OMMER_ROOT_HASH}; use alloy_primitives::{Address, Bytes, Sealable, U256}; -use eth_sparse_mpt::SparseTrieSharedCache; -use reth_db::Database; +use builders::mock_block_building_helper::MockRootHasher; use reth_primitives::BlockBody; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use crate::{ primitives::{Order, OrderId, SimValue, SimulatedOrder, TransactionSignedEcRecoveredWithBlobs}, - roothash::{calculate_state_root, RootHashConfig, RootHashError}, + provider::RootHasher, + roothash::RootHashError, utils::{a2r_withdrawal, calc_gas_limit, timestamp_as_u64, Signer}, }; use ahash::HashSet; @@ -84,7 +83,7 @@ pub struct BlockBuildingContext { pub excess_blob_gas: Option, /// Version of the EVM that we are going to use pub spec_id: SpecId, - pub shared_sparse_mpt_cache: SparseTrieSharedCache, + pub root_hasher: Arc, } impl BlockBuildingContext { @@ -100,6 +99,7 @@ impl BlockBuildingContext { prefer_gas_limit: Option, extra_data: Vec, spec_id: Option, + root_hasher: Arc, ) -> Option { let attributes = EthPayloadBuilderAttributes::try_new( attributes.data.parent_block_hash, @@ -162,10 +162,11 @@ impl BlockBuildingContext { extra_data, excess_blob_gas, spec_id, - shared_sparse_mpt_cache: Default::default(), + root_hasher, }) } + #[allow(clippy::too_many_arguments)] /// `from_block_data` is used to create `BlockBuildingContext` from onchain block for backtest purposes /// spec_id None: we use the SpecId for the block. /// Note: We calculate SpecId based on the current block instead of the parent block so this will break for the blocks +-1 relative to the fork @@ -177,6 +178,7 @@ impl BlockBuildingContext { coinbase: Address, suggested_fee_recipient: Address, builder_signer: Option, + root_hasher: Arc, ) -> BlockBuildingContext { let block_number = onchain_block.header.number; @@ -247,7 +249,7 @@ impl BlockBuildingContext { extra_data: Vec::new(), excess_blob_gas: onchain_block.header.excess_blob_gas, spec_id, - shared_sparse_mpt_cache: Default::default(), + root_hasher, } } @@ -263,6 +265,7 @@ impl BlockBuildingContext { Default::default(), Default::default(), Default::default(), + Arc::new(MockRootHasher {}), ) } @@ -596,20 +599,11 @@ impl PartialBlock { /// Mostly based on reth's (v1.1.1) default_ethereum_payload_builder. #[allow(clippy::too_many_arguments)] - pub fn finalize( + pub fn finalize( self, state: &mut BlockState, ctx: &BlockBuildingContext, - provider: P, - root_hash_config: RootHashConfig, - ) -> Result - where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, - { + ) -> Result { let requests = if ctx .chain_spec .is_prague_active_at_timestamp(ctx.attributes.timestamp()) @@ -689,13 +683,7 @@ impl PartialBlock { // calculate the state root let start = Instant::now(); - let state_root = calculate_state_root( - provider, - ctx.attributes.parent, - &execution_outcome, - ctx.shared_sparse_mpt_cache.clone(), - root_hash_config, - )?; + let state_root = ctx.root_hasher.state_root(&execution_outcome)?; let root_hash_time = start.elapsed(); // create the block header diff --git a/crates/rbuilder/src/building/sim.rs b/crates/rbuilder/src/building/sim.rs index 3eb885b9..26c6ec05 100644 --- a/crates/rbuilder/src/building/sim.rs +++ b/crates/rbuilder/src/building/sim.rs @@ -5,6 +5,7 @@ use super::{ use crate::{ building::{BlockBuildingContext, BlockState, CriticalCommitOrderError}, primitives::{Order, OrderId, SimValue, SimulatedOrder}, + provider::StateProviderFactory, utils::{NonceCache, NonceCacheRef}, }; use ahash::{HashMap, HashSet}; @@ -12,7 +13,7 @@ use alloy_primitives::{Address, B256}; use rand::seq::SliceRandom; use reth::revm::cached::CachedReads; use reth_errors::ProviderError; -use reth_provider::{StateProvider, StateProviderFactory}; +use reth_provider::StateProvider; use std::{ cmp::{max, min, Ordering}, collections::hash_map::Entry, @@ -88,7 +89,7 @@ enum OrderNonceState { impl

SimTree

where - P: StateProviderFactory + Clone + 'static, + P: StateProviderFactory, { pub fn new(provider: P, parent_block: B256) -> Self { let nonce_cache = NonceCache::new(provider, parent_block); @@ -314,7 +315,7 @@ pub fn simulate_all_orders_with_sim_tree

( randomize_insertion: bool, ) -> Result<(Vec, Vec), CriticalCommitOrderError> where - P: StateProviderFactory + Clone + 'static, + P: StateProviderFactory + Clone, { let mut sim_tree = SimTree::new(provider.clone(), ctx.attributes.parent); diff --git a/crates/rbuilder/src/building/testing/test_chain_state.rs b/crates/rbuilder/src/building/testing/test_chain_state.rs index 507ca25d..f58e7bb0 100644 --- a/crates/rbuilder/src/building/testing/test_chain_state.rs +++ b/crates/rbuilder/src/building/testing/test_chain_state.rs @@ -1,3 +1,7 @@ +use crate::provider::RootHasher; +use crate::roothash::RootHashConfig; +use crate::utils::RootHasherImpl; +use crate::{building::BlockBuildingContext, utils::Signer}; use ahash::HashSet; use alloy_consensus::{Header, TxEip1559}; use alloy_primitives::{ @@ -17,8 +21,6 @@ use reth_provider::test_utils::{create_test_provider_factory, MockNodeTypesWithD use revm_primitives::SpecId; use std::sync::Arc; -use crate::{building::BlockBuildingContext, utils::Signer}; - #[derive(Debug, Clone, Copy)] pub enum NamedAddr { Builder, @@ -139,6 +141,13 @@ impl TestChainState { } provider.commit()?; } + + let root_hasher = Arc::from(RootHasherImpl::new( + genesis_header.hash(), + RootHashConfig::new(true, false), + provider_factory.clone(), + )); + let ctx = TestBlockContextBuilder::new( block_args, builder.clone(), @@ -146,6 +155,7 @@ impl TestChainState { chain_spec.clone(), blocklisted_address.address, genesis_header.hash(), + root_hasher, ) .build(); @@ -236,6 +246,7 @@ struct TestBlockContextBuilder { blocklist: HashSet

, prefer_gas_limit: Option, use_suggested_fee_recipient_as_coinbase: bool, + root_hasher: Arc, } impl TestBlockContextBuilder { @@ -246,6 +257,7 @@ impl TestBlockContextBuilder { chain_spec: Arc, blocklisted: Address, parent_hash: BlockHash, + root_hasher: Arc, ) -> Self { TestBlockContextBuilder { parent_gas_limit: 30_000_000, @@ -264,6 +276,7 @@ impl TestBlockContextBuilder { prefer_gas_limit: None, use_suggested_fee_recipient_as_coinbase: block_args .use_suggested_fee_recipient_as_coinbase, + root_hasher, } } @@ -315,6 +328,7 @@ impl TestBlockContextBuilder { self.prefer_gas_limit, vec![], Some(SpecId::SHANGHAI), + self.root_hasher, ) .unwrap(); if self.use_suggested_fee_recipient_as_coinbase { diff --git a/crates/rbuilder/src/lib.rs b/crates/rbuilder/src/lib.rs index 7572d559..fdfed100 100644 --- a/crates/rbuilder/src/lib.rs +++ b/crates/rbuilder/src/lib.rs @@ -5,6 +5,7 @@ pub mod integration; pub mod live_builder; pub mod mev_boost; pub mod primitives; +pub mod provider; pub mod roothash; pub mod telemetry; pub mod utils; diff --git a/crates/rbuilder/src/live_builder/base_config.rs b/crates/rbuilder/src/live_builder/base_config.rs index 001b5845..1e45f454 100644 --- a/crates/rbuilder/src/live_builder/base_config.rs +++ b/crates/rbuilder/src/live_builder/base_config.rs @@ -3,6 +3,7 @@ use crate::{ building::builders::UnfinishedBlockBuildingSinkFactory, live_builder::{order_input::OrderInputConfig, LiveBuilder}, + provider::StateProviderFactory, roothash::RootHashConfig, telemetry::{setup_reloadable_tracing_subscriber, LoggerConfig}, utils::{http_provider, BoxedProvider, ProviderFactoryReopener, Signer}, @@ -13,13 +14,11 @@ use eyre::{eyre, Context}; use jsonrpsee::RpcModule; use reth::chainspec::chain_value_parser; use reth_chainspec::ChainSpec; -use reth_db::{Database, DatabaseEnv}; +use reth_db::DatabaseEnv; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; use reth_primitives::StaticFileSegment; -use reth_provider::{ - DatabaseProviderFactory, HeaderProvider, StateProviderFactory, StaticFileProviderFactory, -}; +use reth_provider::StaticFileProviderFactory; use serde::{Deserialize, Deserializer}; use serde_with::{serde_as, DeserializeAs}; use std::{ @@ -170,22 +169,21 @@ impl BaseConfig { } /// Allows instantiating a [`LiveBuilder`] with an existing provider factory - pub async fn create_builder_with_provider_factory( + pub async fn create_builder_with_provider_factory( &self, cancellation_token: tokio_util::sync::CancellationToken, sink_factory: Box, slot_source: SlotSourceType, provider: P, - ) -> eyre::Result> + ) -> eyre::Result> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory + StateProviderFactory + HeaderProvider + Clone, + P: StateProviderFactory, SlotSourceType: SlotSource, { let order_input_config = OrderInputConfig::from_config(self)?; let (orderpool_sender, orderpool_receiver) = mpsc::channel(order_input_config.input_channel_buffer_size); - Ok(LiveBuilder:: { + Ok(LiveBuilder:: { watchdog_timeout: self.watchdog_timeout(), error_storage_path: self.error_storage_path.clone(), simulation_threads: self.simulation_threads, @@ -242,6 +240,7 @@ impl BaseConfig { self.reth_static_files_path.as_deref(), self.chain_spec()?, false, + Some(self.live_root_hash_config()?), ) } @@ -251,7 +250,7 @@ impl BaseConfig { "root_hash_compare_sparse_trie can't be set without root_hash_use_sparse_trie" ); } - Ok(RootHashConfig::live_config( + Ok(RootHashConfig::new( self.root_hash_use_sparse_trie, self.root_hash_compare_sparse_trie, )) @@ -435,12 +434,14 @@ where } /// Open reth db and DB should be opened once per process but it can be cloned and moved to different threads. +/// root_hash_config None -> MockRootHasher used pub fn create_provider_factory( reth_datadir: Option<&Path>, reth_db_path: Option<&Path>, reth_static_files_path: Option<&Path>, chain_spec: Arc, rw: bool, + root_hash_config: Option, ) -> eyre::Result>>> { // shellexpand the reth datadir let reth_datadir = if let Some(reth_datadir) = reth_datadir { @@ -474,7 +475,7 @@ pub fn create_provider_factory( }; let provider_factory_reopener = - ProviderFactoryReopener::new(db, chain_spec, reth_static_files_path)?; + ProviderFactoryReopener::new(db, chain_spec, reth_static_files_path, root_hash_config)?; if provider_factory_reopener .provider_factory_unchecked() @@ -577,6 +578,7 @@ mod test { reth_static_files_path.as_deref(), Default::default(), true, + None, ); if *should_succeed { diff --git a/crates/rbuilder/src/live_builder/block_output/bidding/wallet_balance_watcher.rs b/crates/rbuilder/src/live_builder/block_output/bidding/wallet_balance_watcher.rs index 682c423a..e6302782 100644 --- a/crates/rbuilder/src/live_builder/block_output/bidding/wallet_balance_watcher.rs +++ b/crates/rbuilder/src/live_builder/block_output/bidding/wallet_balance_watcher.rs @@ -1,14 +1,13 @@ use std::time::Duration; use alloy_primitives::{utils::format_ether, Address, BlockNumber, U256}; -use reth::providers::{HeaderProvider, ProviderError}; -use reth_provider::StateProviderFactory; +use reth::providers::ProviderError; use time::{error, OffsetDateTime}; use tracing::{error, info, warn}; -use crate::telemetry::{add_subsidy_value, inc_subsidized_blocks}; - use super::interfaces::LandedBlockInfo; +use crate::provider::StateProviderFactory; +use crate::telemetry::{add_subsidy_value, inc_subsidized_blocks}; /// Allows to monitor the evolution of our wallet for the landed blocks. /// It's useful for bidders to detect profit and subsidies. @@ -57,7 +56,7 @@ impl BlockInfo { impl

WalletBalanceWatcher

where - P: StateProviderFactory + HeaderProvider, + P: StateProviderFactory, { /// Creates a WalletBalanceWatcher pre-analyzing a window of init_window_size size. pub fn new( diff --git a/crates/rbuilder/src/live_builder/block_output/block_sealing_bidder_factory.rs b/crates/rbuilder/src/live_builder/block_output/block_sealing_bidder_factory.rs index 80f855ce..90af1eec 100644 --- a/crates/rbuilder/src/live_builder/block_output/block_sealing_bidder_factory.rs +++ b/crates/rbuilder/src/live_builder/block_output/block_sealing_bidder_factory.rs @@ -1,9 +1,9 @@ use crate::{ building::builders::{UnfinishedBlockBuildingSink, UnfinishedBlockBuildingSinkFactory}, live_builder::payload_events::MevBoostSlotData, + provider::StateProviderFactory, }; use alloy_primitives::U256; -use reth_provider::{HeaderProvider, StateProviderFactory}; use std::{fmt::Debug, sync::Arc}; use tracing::error; @@ -81,7 +81,7 @@ impl BidValueObs for SlotBidderToBidValueObs { impl

UnfinishedBlockBuildingSinkFactory for BlockSealingBidderFactory

where - P: StateProviderFactory + HeaderProvider, + P: StateProviderFactory, { fn create_sink( &mut self, diff --git a/crates/rbuilder/src/live_builder/building/mod.rs b/crates/rbuilder/src/live_builder/building/mod.rs index b8bd70be..0db6ecfc 100644 --- a/crates/rbuilder/src/live_builder/building/mod.rs +++ b/crates/rbuilder/src/live_builder/building/mod.rs @@ -1,4 +1,4 @@ -use std::{cell::RefCell, marker::PhantomData, rc::Rc, sync::Arc, thread, time::Duration}; +use std::{cell::RefCell, rc::Rc, sync::Arc, thread, time::Duration}; use crate::{ building::{ @@ -10,10 +10,8 @@ use crate::{ }, live_builder::{payload_events::MevBoostSlotData, simulation::SlotOrderSimResults}, primitives::{OrderId, SimulatedOrder}, - roothash::run_trie_prefetcher, + provider::StateProviderFactory, }; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use revm_primitives::Address; use tokio::sync::{broadcast, mpsc}; use tokio_util::sync::CancellationToken; @@ -28,28 +26,23 @@ use super::{ }; #[derive(Debug)] -pub struct BlockBuildingPool { +pub struct BlockBuildingPool

{ provider: P, - builders: Vec>>, + builders: Vec>>, sink_factory: Box, orderpool_subscriber: order_input::OrderPoolSubscriber, order_simulation_pool: OrderSimulationPool

, run_sparse_trie_prefetcher: bool, sbundle_merger_selected_signers: Arc>, - phantom: PhantomData, } -impl BlockBuildingPool +impl

BlockBuildingPool

where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { pub fn new( provider: P, - builders: Vec>>, + builders: Vec>>, sink_factory: Box, orderpool_subscriber: order_input::OrderPoolSubscriber, order_simulation_pool: OrderSimulationPool

, @@ -64,7 +57,6 @@ where order_simulation_pool, run_sparse_trie_prefetcher, sbundle_merger_selected_signers, - phantom: PhantomData, } } @@ -138,16 +130,9 @@ where if self.run_sparse_trie_prefetcher { let input = broadcast_input.subscribe(); - let provider = self.provider.clone(); + tokio::task::spawn_blocking(move || { - run_trie_prefetcher( - ctx.attributes.parent, - ctx.shared_sparse_mpt_cache, - provider, - input, - cancel.clone(), - ); - debug!(block = block_number, "Stopped trie prefetcher job"); + ctx.root_hasher.run_prefetcher(input, cancel); }); } diff --git a/crates/rbuilder/src/live_builder/cli.rs b/crates/rbuilder/src/live_builder/cli.rs index 1e2a0150..5bde1f7d 100644 --- a/crates/rbuilder/src/live_builder/cli.rs +++ b/crates/rbuilder/src/live_builder/cli.rs @@ -2,8 +2,6 @@ use std::path::PathBuf; use clap::Parser; use reth::revm::cached::CachedReads; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory}; use serde::de::DeserializeOwned; use std::fmt::Debug; use sysperf::{format_results, gather_system_info, run_all_benchmarks}; @@ -15,6 +13,7 @@ use crate::{ live_builder::{ base_config::load_config_toml_and_env, payload_events::MevBoostSlotDataGenerator, }, + provider::StateProviderFactory, telemetry, utils::{bls::generate_random_bls_address, build_info::Version}, }; @@ -53,33 +52,23 @@ pub trait LiveBuilderConfig: Debug + DeserializeOwned + Sync { /// Create a concrete builder /// /// Desugared from async to future to keep clippy happy - fn new_builder( + fn new_builder

( &self, provider: P, cancellation_token: CancellationToken, - ) -> impl std::future::Future>> - + Send + ) -> impl std::future::Future>> + Send where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static; + P: StateProviderFactory + Clone + 'static; /// Patch until we have a unified way of backtesting using the exact algorithms we use on the LiveBuilder. /// building_algorithm_name will come from the specific configuration. - fn build_backtest_block( + fn build_backtest_block

( &self, building_algorithm_name: &str, input: BacktestSimulateBlockInput<'_, P>, ) -> eyre::Result<(Block, CachedReads)> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static; + P: StateProviderFactory + Clone + 'static; } /// print_version_info func that will be called on command Cli::Version diff --git a/crates/rbuilder/src/live_builder/config.rs b/crates/rbuilder/src/live_builder/config.rs index 25ab88bc..8cea1290 100644 --- a/crates/rbuilder/src/live_builder/config.rs +++ b/crates/rbuilder/src/live_builder/config.rs @@ -32,6 +32,7 @@ use crate::{ }, mev_boost::BLSBlockSigner, primitives::mev_boost::{MevBoostRelay, RelayConfig}, + provider::StateProviderFactory, roothash::RootHashConfig, utils::{build_info::rbuilder_version, ProviderFactoryReopener, Signer}, validation_api_client::ValidationAPIClient, @@ -49,14 +50,11 @@ use eyre::Context; use lazy_static::lazy_static; use reth::revm::cached::CachedReads; use reth_chainspec::{Chain, ChainSpec, NamedChain}; -use reth_db::{Database, DatabaseEnv}; +use reth_db::DatabaseEnv; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; use reth_primitives::StaticFileSegment; -use reth_provider::{ - BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory, - StaticFileProviderFactory, -}; +use reth_provider::StaticFileProviderFactory; use serde::Deserialize; use serde_with::{serde_as, OneOrMany}; use std::collections::HashMap; @@ -325,18 +323,13 @@ impl LiveBuilderConfig for Config { fn base_config(&self) -> &BaseConfig { &self.base_config } - async fn new_builder( + async fn new_builder

( &self, provider: P, cancellation_token: tokio_util::sync::CancellationToken, - ) -> eyre::Result> + ) -> eyre::Result> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let (sink_sealed_factory, relays) = self.l1_config.create_relays_sealed_sink_factory( self.base_config.chain_spec()?, @@ -374,8 +367,7 @@ impl LiveBuilderConfig for Config { provider, ) .await?; - let root_hash_config = self.base_config.live_root_hash_config()?; - let builders = create_builders(self.live_builders()?, root_hash_config); + let builders = create_builders(self.live_builders()?); Ok(live_builder.with_builders(builders)) } @@ -383,17 +375,13 @@ impl LiveBuilderConfig for Config { rbuilder_version() } - fn build_backtest_block( + fn build_backtest_block

( &self, building_algorithm_name: &str, input: BacktestSimulateBlockInput<'_, P>, ) -> eyre::Result<(Block, CachedReads)> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { let builder_cfg = self.builder(building_algorithm_name)?; match builder_cfg.builder { @@ -401,7 +389,7 @@ impl LiveBuilderConfig for Config { crate::building::builders::ordering_builder::backtest_simulate_block(config, input) } SpecificBuilderConfig::ParallelBuilder(config) => { - parallel_build_backtest::(input, config) + parallel_build_backtest::

(input, config) } } } @@ -505,6 +493,7 @@ pub fn create_provider_factory( reth_db_path: Option<&Path>, reth_static_files_path: Option<&Path>, chain_spec: Arc, + root_hash_config: Option, ) -> eyre::Result>>> { let reth_db_path = match (reth_db_path, reth_datadir) { (Some(reth_db_path), _) => PathBuf::from(reth_db_path), @@ -523,7 +512,7 @@ pub fn create_provider_factory( }; let provider_factory_reopener = - ProviderFactoryReopener::new(db, chain_spec, reth_static_files_path)?; + ProviderFactoryReopener::new(db, chain_spec, reth_static_files_path, root_hash_config)?; if provider_factory_reopener .provider_factory_unchecked() @@ -548,41 +537,24 @@ pub fn coinbase_signer_from_secret_key(secret_key: &str) -> eyre::Result Ok(Signer::try_from_secret(secret_key)?) } -pub fn create_builders( - configs: Vec, - root_hash_config: RootHashConfig, -) -> Vec>> +pub fn create_builders

(configs: Vec) -> Vec>> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { - configs - .into_iter() - .map(|cfg| create_builder(cfg, &root_hash_config)) - .collect() + configs.into_iter().map(|cfg| create_builder(cfg)).collect() } -fn create_builder( - cfg: BuilderConfig, - root_hash_config: &RootHashConfig, -) -> Arc> +fn create_builder

(cfg: BuilderConfig) -> Arc> where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, { match cfg.builder { - SpecificBuilderConfig::OrderingBuilder(order_cfg) => Arc::new( - OrderingBuildingAlgorithm::new(root_hash_config.clone(), order_cfg, cfg.name), - ), - SpecificBuilderConfig::ParallelBuilder(parallel_cfg) => Arc::new( - ParallelBuildingAlgorithm::new(root_hash_config.clone(), parallel_cfg, cfg.name), - ), + SpecificBuilderConfig::OrderingBuilder(order_cfg) => { + Arc::new(OrderingBuildingAlgorithm::new(order_cfg, cfg.name)) + } + SpecificBuilderConfig::ParallelBuilder(parallel_cfg) => { + Arc::new(ParallelBuildingAlgorithm::new(parallel_cfg, cfg.name)) + } } } diff --git a/crates/rbuilder/src/live_builder/mod.rs b/crates/rbuilder/src/live_builder/mod.rs index f2754c3d..4d9613ea 100644 --- a/crates/rbuilder/src/live_builder/mod.rs +++ b/crates/rbuilder/src/live_builder/mod.rs @@ -18,6 +18,7 @@ use crate::{ simulation::OrderSimulationPool, watchdog::spawn_watchdog_thread, }, + provider::StateProviderFactory, telemetry::inc_active_slots, utils::{ error_storage::spawn_error_storage_writer, provider_head_state::ProviderHeadState, Signer, @@ -31,10 +32,7 @@ use eyre::Context; use jsonrpsee::RpcModule; use order_input::ReplaceableOrderPoolCommand; use payload_events::MevBoostSlotData; -use reth::providers::HeaderProvider; use reth_chainspec::ChainSpec; -use reth_db::Database; -use reth_provider::{BlockReader, DatabaseProviderFactory, StateProviderFactory}; use std::{cmp::min, fmt::Debug, path::PathBuf, sync::Arc, time::Duration}; use time::OffsetDateTime; use tokio::sync::mpsc; @@ -88,10 +86,9 @@ const CLEAN_TASKS_CHANNEL_SIZE: usize = 10; /// # Usage /// Create and run() #[derive(Debug)] -pub struct LiveBuilder +pub struct LiveBuilder where - DB: Database + Clone + 'static, - P: StateProviderFactory + Clone, + P: StateProviderFactory, BlocksSourceType: SlotSource, { pub watchdog_timeout: Option, @@ -111,7 +108,7 @@ where pub global_cancellation: CancellationToken, pub sink_factory: Box, - pub builders: Vec>>, + pub builders: Vec>>, pub extra_rpc: RpcModule<()>, /// Notify rbuilder of new [`ReplaceableOrderPoolCommand`] flow via this channel. @@ -120,21 +117,16 @@ where pub sbundle_merger_selected_signers: Arc>, } -impl LiveBuilder +impl LiveBuilder where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory - + HeaderProvider - + Clone - + 'static, + P: StateProviderFactory + Clone + 'static, BlocksSourceType: SlotSource, { pub fn with_extra_rpc(self, extra_rpc: RpcModule<()>) -> Self { Self { extra_rpc, ..self } } - pub fn with_builders(self, builders: Vec>>) -> Self { + pub fn with_builders(self, builders: Vec>>) -> Self { Self { builders, ..self } } @@ -262,6 +254,8 @@ where inc_active_slots(); + let root_hasher = Arc::from(self.provider.root_hasher(payload.parent_block_hash())); + if let Some(block_ctx) = BlockBuildingContext::from_attributes( payload.payload_attributes_event.clone(), &parent_header, @@ -271,6 +265,7 @@ where Some(payload.suggested_gas_limit), self.extra_data.clone(), None, + root_hasher, ) { builder_pool.start_block_building( payload, @@ -312,11 +307,11 @@ where async fn wait_for_block_header

( block: B256, slot_time: OffsetDateTime, - provider: P, + provider: &P, timings: &TimingsConfig, ) -> eyre::Result

where - P: HeaderProvider, + P: StateProviderFactory, { let deadline = slot_time + timings.block_header_deadline_delta; while OffsetDateTime::now_utc() < deadline { diff --git a/crates/rbuilder/src/live_builder/order_input/mod.rs b/crates/rbuilder/src/live_builder/order_input/mod.rs index 53fcffe1..1660e726 100644 --- a/crates/rbuilder/src/live_builder/order_input/mod.rs +++ b/crates/rbuilder/src/live_builder/order_input/mod.rs @@ -11,20 +11,14 @@ use self::{ orderpool::{OrderPool, OrderPoolSubscriptionId}, replaceable_order_sink::ReplaceableOrderSink, }; -use crate::{ - primitives::{serialize::CancelShareBundle, BundleReplacementKey, Order}, - telemetry::{set_current_block, set_ordepool_count}, -}; +use crate::primitives::{serialize::CancelShareBundle, BundleReplacementKey, Order}; +use crate::provider::StateProviderFactory; +use crate::telemetry::{set_current_block, set_ordepool_count}; use alloy_consensus::Header; use jsonrpsee::RpcModule; use parking_lot::Mutex; -use reth_provider::StateProviderFactory; -use std::{ - net::Ipv4Addr, - path::{Path, PathBuf}, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{net::Ipv4Addr, path::PathBuf, sync::Arc, time::Duration}; +use std::{path::Path, time::Instant}; use tokio::{sync::mpsc, task::JoinHandle}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; diff --git a/crates/rbuilder/src/live_builder/simulation/mod.rs b/crates/rbuilder/src/live_builder/simulation/mod.rs index 05027dc0..a39612e8 100644 --- a/crates/rbuilder/src/live_builder/simulation/mod.rs +++ b/crates/rbuilder/src/live_builder/simulation/mod.rs @@ -8,11 +8,11 @@ use crate::{ }, live_builder::order_input::orderpool::OrdersForBlock, primitives::{OrderId, SimulatedOrder}, + provider::StateProviderFactory, utils::{gen_uid, Signer}, }; use ahash::HashMap; use parking_lot::Mutex; -use reth_provider::StateProviderFactory; use simulation_job::SimulationJob; use std::sync::Arc; use tokio::{sync::mpsc, task::JoinHandle}; @@ -185,9 +185,11 @@ mod tests { // Create simulation core let cancel = CancellationToken::new(); - let provider_factory_reopener = - ProviderFactoryReopener::new_from_existing(test_context.provider_factory().clone()) - .unwrap(); + let provider_factory_reopener = ProviderFactoryReopener::new_from_existing( + test_context.provider_factory().clone(), + None, + ) + .unwrap(); let sim_pool = OrderSimulationPool::new(provider_factory_reopener, 4, cancel.clone()); let (order_sender, order_receiver) = mpsc::unbounded_channel(); diff --git a/crates/rbuilder/src/live_builder/simulation/sim_worker.rs b/crates/rbuilder/src/live_builder/simulation/sim_worker.rs index 4c572dc2..a0412c4d 100644 --- a/crates/rbuilder/src/live_builder/simulation/sim_worker.rs +++ b/crates/rbuilder/src/live_builder/simulation/sim_worker.rs @@ -4,12 +4,12 @@ use crate::{ simulate_order, BlockState, }, live_builder::simulation::CurrentSimulationContexts, + provider::StateProviderFactory, telemetry, telemetry::add_sim_thread_utilisation_timings, }; use parking_lot::Mutex; use reth::revm::cached::CachedReads; -use reth_provider::StateProviderFactory; use std::{ sync::Arc, thread::sleep, diff --git a/crates/rbuilder/src/live_builder/simulation/simulation_job.rs b/crates/rbuilder/src/live_builder/simulation/simulation_job.rs index 3df589ca..ebe9ed4e 100644 --- a/crates/rbuilder/src/live_builder/simulation/simulation_job.rs +++ b/crates/rbuilder/src/live_builder/simulation/simulation_job.rs @@ -4,10 +4,10 @@ use crate::{ building::sim::{SimTree, SimulatedResult, SimulationRequest}, live_builder::order_input::order_sink::OrderPoolCommand, primitives::{Order, OrderId}, + provider::StateProviderFactory, }; use ahash::HashSet; use alloy_primitives::utils::format_ether; -use reth_provider::StateProviderFactory; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; @@ -58,7 +58,7 @@ pub struct SimulationJob

{ impl

SimulationJob

where - P: StateProviderFactory + Clone + 'static, + P: StateProviderFactory, { pub fn new( block_cancellation: CancellationToken, diff --git a/crates/rbuilder/src/provider/mod.rs b/crates/rbuilder/src/provider/mod.rs new file mode 100644 index 00000000..d4e202fd --- /dev/null +++ b/crates/rbuilder/src/provider/mod.rs @@ -0,0 +1,51 @@ +use crate::live_builder::simulation::SimulatedOrderCommand; +use crate::roothash::RootHashError; +use alloy_consensus::Header; +use alloy_primitives::{BlockHash, BlockNumber, B256}; +use reth::providers::ExecutionOutcome; +use reth_errors::ProviderResult; +use reth_provider::StateProviderBox; +use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; + +pub mod reth_prov; + +/// Main trait to interact with the chain data. +/// Allows to create different backends for chain data access without implementing lots of interfaces as would happen with reth_provider::StateProviderFactory +/// since it only asks for what we really use. +pub trait StateProviderFactory: Send + Sync { + fn latest(&self) -> ProviderResult; + + fn history_by_block_number(&self, block: BlockNumber) -> ProviderResult; + + fn history_by_block_hash(&self, block: BlockHash) -> ProviderResult; + + fn header(&self, block_hash: &BlockHash) -> ProviderResult>; + + fn block_hash(&self, number: BlockNumber) -> ProviderResult>; + + fn best_block_number(&self) -> ProviderResult; + + fn header_by_number(&self, num: u64) -> ProviderResult>; + + fn last_block_number(&self) -> ProviderResult; + + fn root_hasher(&self, parent_hash: B256) -> Box; +} + +/// trait that computes the roothash for a new block assuming a predefine parent block (given in StateProviderFactory::root_hasher) +/// Ideally, it caches information in each roothash is computes (state_root) so the next one is faster. +/// Before using all run_prefetcher to allow the RootHasher start a prefetcher task that will pre cache root state trie nodes +/// based on what it sees on the simulations. +pub trait RootHasher: std::fmt::Debug + Send + Sync { + /// Must be called once before using. + /// This is too specific and prone to error (you may forget to call it), maybe it's a better idea to pass this to StateProviderFactory::root_hasher and let each RootHasher decide what to do? + fn run_prefetcher( + &self, + simulated_orders: broadcast::Receiver, + cancel: CancellationToken, + ); + + /// State root for changes outcome on top of parent block. + fn state_root(&self, outcome: &ExecutionOutcome) -> Result; +} diff --git a/crates/rbuilder/src/provider/reth_prov.rs b/crates/rbuilder/src/provider/reth_prov.rs new file mode 100644 index 00000000..a7fce0c4 --- /dev/null +++ b/crates/rbuilder/src/provider/reth_prov.rs @@ -0,0 +1,71 @@ +use crate::roothash::RootHashConfig; +use crate::utils::RootHasherImpl; +use alloy_consensus::Header; +use alloy_primitives::{BlockHash, BlockNumber, B256}; +use reth_errors::ProviderResult; +use reth_provider::StateProviderBox; +use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider}; + +use super::{RootHasher, StateProviderFactory}; + +/// StateProviderFactory based on a reth traits. +#[derive(Clone)] +pub struct StateProviderFactoryFromRethProvider

{ + provider: P, + config: RootHashConfig, +} + +impl

StateProviderFactoryFromRethProvider

{ + pub fn new(provider: P, config: RootHashConfig) -> Self { + Self { provider, config } + } +} + +impl

StateProviderFactory for StateProviderFactoryFromRethProvider

+where + P: DatabaseProviderFactory + + reth_provider::StateProviderFactory + + HeaderProvider + + Clone + + 'static, +{ + fn latest(&self) -> ProviderResult { + self.provider.latest() + } + + fn history_by_block_number(&self, block: BlockNumber) -> ProviderResult { + self.provider.history_by_block_number(block) + } + + fn history_by_block_hash(&self, block: BlockHash) -> ProviderResult { + self.provider.history_by_block_hash(block) + } + + fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + self.provider.header(block_hash) + } + + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.provider.block_hash(number) + } + + fn best_block_number(&self) -> ProviderResult { + self.provider.best_block_number() + } + + fn header_by_number(&self, num: u64) -> ProviderResult> { + self.provider.header_by_number(num) + } + + fn last_block_number(&self) -> ProviderResult { + self.provider.last_block_number() + } + + fn root_hasher(&self, parent_hash: B256) -> Box { + Box::new(RootHasherImpl::new( + parent_hash, + self.config.clone(), + self.provider.clone(), + )) + } +} diff --git a/crates/rbuilder/src/roothash/mod.rs b/crates/rbuilder/src/roothash/mod.rs index b8ece6eb..0293e499 100644 --- a/crates/rbuilder/src/roothash/mod.rs +++ b/crates/rbuilder/src/roothash/mod.rs @@ -22,9 +22,6 @@ pub enum RootHashMode { /// Makes correct root hash calculation on the incorrect parent state. /// It can be used for benchmarks. IgnoreParentHash, - /// Don't calculate root hash. - /// It can be used for backtest. - SkipRootHash, } #[derive(Debug, thiserror::Error)] @@ -61,15 +58,7 @@ pub struct RootHashConfig { } impl RootHashConfig { - pub fn skip_root_hash() -> Self { - Self { - mode: RootHashMode::SkipRootHash, - use_sparse_trie: false, - compare_sparse_trie_output: false, - } - } - - pub fn live_config(use_sparse_trie: bool, compare_sparse_trie_output: bool) -> Self { + pub fn new(use_sparse_trie: bool, compare_sparse_trie_output: bool) -> Self { Self { mode: RootHashMode::CorrectRoot, use_sparse_trie, @@ -100,7 +89,7 @@ pub fn calculate_state_root

( parent_hash: B256, outcome: &ExecutionOutcome, sparse_trie_shared_cache: SparseTrieSharedCache, - config: RootHashConfig, + config: &RootHashConfig, ) -> Result where P: DatabaseProviderFactory + Send + Sync + Clone + 'static, @@ -109,9 +98,6 @@ where RootHashMode::CorrectRoot => ConsistentDbView::new(provider, Some(parent_hash)), RootHashMode::IgnoreParentHash => ConsistentDbView::new_with_latest_tip(provider) .map_err(ParallelStateRootError::Provider)?, - RootHashMode::SkipRootHash => { - return Ok(B256::ZERO); - } }; let reference_root_hash = if config.compare_sparse_trie_output { diff --git a/crates/rbuilder/src/utils/mod.rs b/crates/rbuilder/src/utils/mod.rs index 6635655b..df15b409 100644 --- a/crates/rbuilder/src/utils/mod.rs +++ b/crates/rbuilder/src/utils/mod.rs @@ -29,7 +29,7 @@ use alloy_eips::eip2718::Encodable2718; pub use noncer::{NonceCache, NonceCacheRef}; pub use provider_factory_reopen::{ check_block_hash_reader_health, is_provider_factory_health_error, HistoricalBlockError, - ProviderFactoryReopener, + ProviderFactoryReopener, RootHasherImpl, }; use reth_chainspec::ChainSpec; use reth_evm_ethereum::revm_spec_by_timestamp_after_merge; diff --git a/crates/rbuilder/src/utils/noncer.rs b/crates/rbuilder/src/utils/noncer.rs index 2fa7b218..b2e0c2f3 100644 --- a/crates/rbuilder/src/utils/noncer.rs +++ b/crates/rbuilder/src/utils/noncer.rs @@ -1,9 +1,9 @@ +use crate::provider::StateProviderFactory; use ahash::HashMap; use alloy_primitives::{Address, B256}; use parking_lot::Mutex; use reth::providers::StateProviderBox; use reth_errors::ProviderResult; -use reth_provider::StateProviderFactory; use std::sync::Arc; /// Struct to get nonces for Addresses, caching the results. diff --git a/crates/rbuilder/src/utils/provider_factory_reopen.rs b/crates/rbuilder/src/utils/provider_factory_reopen.rs index fafd9817..7d7adfa6 100644 --- a/crates/rbuilder/src/utils/provider_factory_reopen.rs +++ b/crates/rbuilder/src/utils/provider_factory_reopen.rs @@ -1,25 +1,27 @@ +use crate::building::builders::mock_block_building_helper::MockRootHasher; +use crate::live_builder::simulation::SimulatedOrderCommand; +use crate::provider::{RootHasher, StateProviderFactory}; +use crate::roothash::{calculate_state_root, run_trie_prefetcher, RootHashConfig, RootHashError}; use crate::telemetry::{inc_provider_bad_reopen_counter, inc_provider_reopen_counter}; use alloy_consensus::Header; -use alloy_eips::{BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{BlockHash, BlockNumber}; +use eth_sparse_mpt::reth_sparse_trie::SparseTrieSharedCache; use parking_lot::{Mutex, RwLock}; +use reth::providers::ExecutionOutcome; use reth::providers::{BlockHashReader, ChainSpecProvider, ProviderFactory}; -use reth_chainspec::ChainInfo; -use reth_db::{Database, DatabaseError}; +use reth_db::DatabaseError; use reth_errors::{ProviderError, ProviderResult, RethResult}; use reth_node_api::NodeTypesWithDB; -use reth_primitives::SealedHeader; use reth_provider::{ providers::{ProviderNodeTypes, StaticFileProvider}, - BlockIdReader, BlockNumReader, DatabaseProvider, DatabaseProviderFactory, DatabaseProviderRO, - HeaderProvider, StateProviderBox, StateProviderFactory, StaticFileProviderFactory, -}; -use revm_primitives::{B256, U256}; -use std::{ - ops::{DerefMut, RangeBounds}, - path::PathBuf, - sync::Arc, + BlockNumReader, HeaderProvider, StateProviderBox, StaticFileProviderFactory, }; +use reth_provider::{BlockReader, DatabaseProviderFactory}; +use revm_primitives::B256; +use std::ops::DerefMut; +use std::{path::PathBuf, sync::Arc}; +use tokio::sync::broadcast; +use tokio_util::sync::CancellationToken; use tracing::debug; /// This struct is used as a workaround for https://github.com/paradigmxyz/reth/issues/7836 @@ -35,13 +37,17 @@ pub struct ProviderFactoryReopener { last_consistent_block: Arc>>, /// Patch to disable checking on test mode. Is ugly but ProviderFactoryReopener should die shortly (5/24/2024). testing_mode: bool, + /// None ->No root hash (MockRootHasher) + root_hash_config: Option, } +/// root_hash_config None -> MockRootHasher used impl ProviderFactoryReopener { pub fn new( db: N::DB, chain_spec: Arc, static_files_path: PathBuf, + root_hash_config: Option, ) -> RethResult { let provider_factory = ProviderFactory::new( db, @@ -53,18 +59,23 @@ impl ProviderFactoryReopener provider_factory: Arc::new(Mutex::new(provider_factory)), chain_spec, static_files_path, + root_hash_config, testing_mode: false, last_consistent_block: Arc::new(RwLock::new(None)), }) } - pub fn new_from_existing(provider_factory: ProviderFactory) -> RethResult { + pub fn new_from_existing( + provider_factory: ProviderFactory, + root_hash_config: Option, + ) -> RethResult { let chain_spec = provider_factory.chain_spec(); let static_files_path = provider_factory.static_file_provider().path().to_path_buf(); Ok(Self { provider_factory: Arc::new(Mutex::new(provider_factory)), chain_spec, static_files_path, + root_hash_config, testing_mode: true, last_consistent_block: Arc::new(RwLock::new(None)), }) @@ -166,93 +177,37 @@ pub fn check_block_hash_reader_health( Ok(()) } -// Implement reth db traits on the ProviderFactoryReopener, allowing generic -// DB access. -// -// ProviderFactory only has access to disk state, therefore cannot implement methods -// that require the blockchain tree (pending state etc.). - -impl DatabaseProviderFactory +impl StateProviderFactory for ProviderFactoryReopener { - /// Database this factory produces providers for. - type DB = N::DB; - /// Provider type returned by the factory. - type Provider = DatabaseProviderRO; - /// Read-write provider type returned by the factory. - type ProviderRW = DatabaseProvider<::TXMut, N>; - - /// Create new read-write database provider. - fn database_provider_rw(&self) -> ProviderResult { - unimplemented!("This method is not supported by ProviderFactoryReopener. We don't write."); - } - - fn database_provider_ro(&self) -> ProviderResult { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.database_provider_ro() - } -} - -impl HeaderProvider for ProviderFactoryReopener { - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.header(block_hash) - } - - fn header_by_number(&self, num: u64) -> ProviderResult> { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.header_by_number(num) - } - - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.header_td(hash) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { + fn latest(&self) -> ProviderResult { let provider = self .check_consistency_and_reopen_if_needed() .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.header_td_by_number(number) + provider.latest() } - fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { + fn history_by_block_number(&self, block: BlockNumber) -> ProviderResult { let provider = self .check_consistency_and_reopen_if_needed() .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.headers_range(range) + provider.history_by_block_number(block) } - fn sealed_header(&self, number: BlockNumber) -> ProviderResult> { + fn history_by_block_hash(&self, block: BlockHash) -> ProviderResult { let provider = self .check_consistency_and_reopen_if_needed() .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.sealed_header(number) + provider.history_by_block_hash(block) } - fn sealed_headers_while( - &self, - range: impl RangeBounds, - predicate: impl FnMut(&SealedHeader) -> bool, - ) -> ProviderResult> { + fn best_block_number(&self) -> ProviderResult { let provider = self .check_consistency_and_reopen_if_needed() .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.sealed_headers_while(range, predicate) + provider.best_block_number() } -} -impl BlockHashReader - for ProviderFactoryReopener -{ fn block_hash(&self, number: BlockNumber) -> ProviderResult> { let provider = self .check_consistency_and_reopen_if_needed() @@ -260,31 +215,18 @@ impl BlockHashReader provider.block_hash(number) } - fn canonical_hashes_range( - &self, - start: BlockNumber, - end: BlockNumber, - ) -> ProviderResult> { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.canonical_hashes_range(start, end) - } -} - -impl BlockNumReader for ProviderFactoryReopener { - fn chain_info(&self) -> ProviderResult { + fn header(&self, block_hash: &BlockHash) -> ProviderResult> { let provider = self .check_consistency_and_reopen_if_needed() .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.chain_info() + provider.header(block_hash) } - fn best_block_number(&self) -> ProviderResult { + fn header_by_number(&self, num: u64) -> ProviderResult> { let provider = self .check_consistency_and_reopen_if_needed() .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.best_block_number() + provider.header_by_number(num) } fn last_block_number(&self) -> ProviderResult { @@ -294,68 +236,74 @@ impl BlockNumReader for Provider provider.last_block_number() } - fn block_number(&self, hash: B256) -> ProviderResult> { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.block_number(hash) + fn root_hasher(&self, parent_hash: B256) -> Box { + if let Some(root_hash_config) = &self.root_hash_config { + let provider = self + .check_consistency_and_reopen_if_needed() + .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string()))) + .unwrap(); + Box::new(RootHasherImpl::new( + parent_hash, + root_hash_config.clone(), + provider, + )) + } else { + Box::new(MockRootHasher {}) + } } } -impl BlockIdReader for ProviderFactoryReopener { - fn pending_block_num_hash(&self) -> ProviderResult> { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); - } - - fn safe_block_num_hash(&self) -> ProviderResult> { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); - } +pub struct RootHasherImpl { + parent_hash: B256, + provider: T, + sparse_trie_shared_cache: SparseTrieSharedCache, + config: RootHashConfig, +} - fn finalized_block_num_hash(&self) -> ProviderResult> { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); +impl RootHasherImpl { + pub fn new(parent_hash: B256, config: RootHashConfig, provider: T) -> Self { + Self { + parent_hash, + provider, + config, + sparse_trie_shared_cache: Default::default(), + } } } -impl StateProviderFactory - for ProviderFactoryReopener +impl RootHasher for RootHasherImpl +where + T: DatabaseProviderFactory + Send + Sync + Clone + 'static, { - fn latest(&self) -> ProviderResult { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.latest() - } - - fn state_by_block_number_or_tag( + fn run_prefetcher( &self, - _number_or_tag: BlockNumberOrTag, - ) -> ProviderResult { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); - } - - fn history_by_block_number(&self, block: BlockNumber) -> ProviderResult { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.history_by_block_number(block) - } - - fn history_by_block_hash(&self, block: BlockHash) -> ProviderResult { - let provider = self - .check_consistency_and_reopen_if_needed() - .map_err(|e| ProviderError::Database(DatabaseError::Other(e.to_string())))?; - provider.history_by_block_hash(block) - } - - fn state_by_block_hash(&self, _block: BlockHash) -> ProviderResult { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); + simulated_orders: broadcast::Receiver, + cancel: CancellationToken, + ) { + run_trie_prefetcher( + self.parent_hash, + self.sparse_trie_shared_cache.clone(), + self.provider.clone(), + simulated_orders, + cancel, + ); } - fn pending(&self) -> ProviderResult { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); + fn state_root(&self, outcome: &ExecutionOutcome) -> Result { + calculate_state_root( + self.provider.clone(), + self.parent_hash, + outcome, + self.sparse_trie_shared_cache.clone(), + &self.config, + ) } +} - fn pending_state_by_hash(&self, _block_hash: B256) -> ProviderResult> { - unimplemented!("This method is not supported by ProviderFactoryReopener. Please consider using a BlockchainProvider."); +impl std::fmt::Debug for RootHasherImpl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RootHasherImpl") + .field("parent_hash", &self.parent_hash) + .finish() } } diff --git a/crates/rbuilder/src/utils/provider_head_state.rs b/crates/rbuilder/src/utils/provider_head_state.rs index a73df854..4d08eaf6 100644 --- a/crates/rbuilder/src/utils/provider_head_state.rs +++ b/crates/rbuilder/src/utils/provider_head_state.rs @@ -1,8 +1,8 @@ //! Mod for gathering info about reth's head. +use crate::provider::StateProviderFactory; use alloy_primitives::BlockNumber; use reth_errors::ProviderResult; -use reth_provider::{BlockHashReader, BlockNumReader}; use revm_primitives::B256; /// For debugging. Results of asking for block number + hash to a BlockHashReader+BlockNumReader @@ -15,7 +15,7 @@ pub struct ProviderHeadStateBlockHash { } impl ProviderHeadStateBlockHash { - pub fn new( + pub fn new( provider: &P, block_number: ProviderResult, ) -> Self { @@ -34,7 +34,7 @@ pub struct ProviderHeadState { } impl ProviderHeadState { - pub fn new(provider: &P) -> Self { + pub fn new(provider: &P) -> Self { Self { last_block: ProviderHeadStateBlockHash::new(provider, provider.last_block_number()), best_block: ProviderHeadStateBlockHash::new(provider, provider.best_block_number()), diff --git a/crates/reth-rbuilder/src/main.rs b/crates/reth-rbuilder/src/main.rs index 1e0b2f20..79be7bc9 100644 --- a/crates/reth-rbuilder/src/main.rs +++ b/crates/reth-rbuilder/src/main.rs @@ -8,10 +8,10 @@ use clap::{Args, Parser}; use rbuilder::{ live_builder::{base_config::load_config_toml_and_env, cli::LiveBuilderConfig, config::Config}, + provider::reth_prov::StateProviderFactoryFromRethProvider, telemetry, }; use reth::{chainspec::EthereumChainSpecParser, cli::Cli}; -use reth_db_api::Database; use reth_node_builder::{ engine_tree_config::{ TreeConfig, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, @@ -21,7 +21,7 @@ use reth_node_builder::{ use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; use reth_provider::{ providers::{BlockchainProvider, BlockchainProvider2}, - BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory, + BlockReader, DatabaseProviderFactory, HeaderProvider, }; use std::{path::PathBuf, process}; use tokio::task; @@ -121,11 +121,10 @@ fn main() { /// Spawns a tokio rbuilder task. /// /// Takes down the entire process if the rbuilder errors or stops. -fn spawn_rbuilder(provider: P, config_path: PathBuf) +fn spawn_rbuilder

(provider: P, config_path: PathBuf) where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory + P: DatabaseProviderFactory + + reth_provider::StateProviderFactory + HeaderProvider + Clone + 'static, @@ -147,8 +146,16 @@ where config.base_config.log_enable_dynamic, ) .await?; - let builder = config.new_builder(provider, Default::default()).await?; + let builder = config + .new_builder( + StateProviderFactoryFromRethProvider::new( + provider, + config.base_config().live_root_hash_config()?, + ), + Default::default(), + ) + .await?; builder.run().await?; Ok::<(), eyre::Error>(()) diff --git a/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/Cargo.toml b/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/Cargo.toml index 7618b468..c6a03034 100644 --- a/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/Cargo.toml +++ b/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/Cargo.toml @@ -10,9 +10,7 @@ rbuilder = { path = "../../../rbuilder" } alloy-primitives.workspace = true alloy-rpc-types-beacon.workspace = true reth-primitives = { workspace = true } -reth-provider = { workspace = true } -reth-db-api = { workspace = true } - +reth-provider = {workspace = true} derive_more = { workspace = true } eyre = { workspace = true } tokio = { workspace = true } @@ -22,7 +20,5 @@ tracing = { workspace = true } [features] optimism = [ "rbuilder/optimism", - "reth-db-api/optimism", "reth-primitives/optimism", - "reth-provider/optimism" ] diff --git a/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/src/lib.rs b/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/src/lib.rs index 716a26a3..202978ff 100644 --- a/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/src/lib.rs +++ b/crates/transaction-pool-bundle-ext/bundle_pool_ops/rbuilder/src/lib.rs @@ -26,11 +26,11 @@ use rbuilder::{ SlotSource, }, primitives::{Bundle, BundleReplacementKey, Order}, + provider::reth_prov::StateProviderFactoryFromRethProvider, telemetry, }; -use reth_db_api::Database; use reth_primitives::TransactionSigned; -use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider, StateProviderFactory}; +use reth_provider::{BlockReader, DatabaseProviderFactory, HeaderProvider}; use tokio::{ sync::{ mpsc::{self, error::SendError}, @@ -89,14 +89,10 @@ impl SlotSource for OurSlotSource { } impl BundlePoolOps { - pub async fn new( - provider: P, - rbuilder_config_path: impl AsRef, - ) -> Result + pub async fn new

(provider: P, rbuilder_config_path: impl AsRef) -> Result where - DB: Database + Clone + 'static, - P: DatabaseProviderFactory - + StateProviderFactory + P: DatabaseProviderFactory + + reth_provider::StateProviderFactory + HeaderProvider + Clone + 'static, @@ -127,16 +123,17 @@ impl BundlePoolOps { build_duration_deadline_ms: None, }), }; - - let builders = create_builders( - vec![builder_strategy], - config.base_config.live_root_hash_config().unwrap(), + let provider = StateProviderFactoryFromRethProvider::new( + provider, + config.base_config().live_root_hash_config()?, ); + let builders = create_builders(vec![builder_strategy]); + // Build and run the process let builder = config .base_config - .create_builder_with_provider_factory::( + .create_builder_with_provider_factory( cancellation_token, Box::new(sink_factory), slot_source,