L1 client failover

Allow the L1 client to be configured with multiple RPC provider URLs. Fail over to the next provider in the list if * two requests fail in a given time period * N requests fail in a row Handle rate limit errors specially by preventing requests to the provider until the rate limit period expires. Closes #2360
EspressoSystems · Jan 3, 2025 · 03b2c6a · 03b2c6a
1 parent a54f777
commit 03b2c6a
Show file tree

Hide file tree

Showing 15 changed files with 487 additions and 94 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/builder/src/bin/permissionless-builder.rs b/builder/src/bin/permissionless-builder.rs
@@ -40,8 +40,13 @@ struct NonPermissionedBuilderOptions {
     eth_account_index: u32,
 
     /// Url we will use for RPC communication with L1.
-    #[clap(long, env = "ESPRESSO_BUILDER_L1_PROVIDER")]
-    l1_provider_url: Url,
+    #[clap(
+        long,
+        env = "ESPRESSO_BUILDER_L1_PROVIDER",
+        value_delimiter = ',',
+        num_args = 1..,
+    )]
+    l1_provider_url: Vec<Url>,
 
     /// Peer nodes use to fetch missing state
     #[clap(long, env = "ESPRESSO_SEQUENCER_STATE_PEERS", value_delimiter = ',')]
@@ -133,7 +138,7 @@ async fn run<V: Versions>(
     opt: NonPermissionedBuilderOptions,
 ) -> anyhow::Result<()> {
     let l1_params = L1Params {
-        url: opt.l1_provider_url,
+        urls: opt.l1_provider_url,
         options: Default::default(),
     };
 

diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs
@@ -46,7 +46,7 @@ pub fn build_instance_state<V: Versions>(
     l1_params: L1Params,
     state_peers: Vec<Url>,
 ) -> NodeState {
-    let l1_client = l1_params.options.connect(l1_params.url);
+    let l1_client = l1_params.options.connect(l1_params.urls);
     NodeState::new(
         u64::MAX, // dummy node ID, only used for debugging
         chain_config,

diff --git a/marketplace-builder/src/bin/marketplace-builder.rs b/marketplace-builder/src/bin/marketplace-builder.rs
@@ -48,8 +48,13 @@ struct NonPermissionedBuilderOptions {
     eth_account_index: u32,
 
     /// Url we will use for RPC communication with L1.
-    #[clap(long, env = "ESPRESSO_BUILDER_L1_PROVIDER")]
-    l1_provider_url: Url,
+    #[clap(
+        long,
+        env = "ESPRESSO_BUILDER_L1_PROVIDER",
+        value_delimiter = ',',
+        num_args = 1..,
+    )]
+    l1_provider_url: Vec<Url>,
 
     /// Peer nodes use to fetch missing state
     #[clap(long, env = "ESPRESSO_SEQUENCER_STATE_PEERS", value_delimiter = ',')]
@@ -140,7 +145,7 @@ async fn run<V: Versions>(
     opt: NonPermissionedBuilderOptions,
 ) -> anyhow::Result<()> {
     let l1_params = L1Params {
-        url: opt.l1_provider_url,
+        urls: opt.l1_provider_url,
         options: Default::default(),
     };
 

diff --git a/marketplace-builder/src/builder.rs b/marketplace-builder/src/builder.rs
@@ -66,7 +66,7 @@ pub fn build_instance_state<V: Versions>(
     l1_params: L1Params,
     state_peers: Vec<Url>,
 ) -> NodeState {
-    let l1_client = l1_params.options.connect(l1_params.url);
+    let l1_client = l1_params.options.connect(l1_params.urls);
     NodeState::new(
         u64::MAX, // dummy node ID, only used for debugging
         chain_config,

diff --git a/sequencer-sqlite/Cargo.lock b/sequencer-sqlite/Cargo.lock
diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs
@@ -11,6 +11,7 @@ use espresso_types::{
 use ethers::types::H160;
 use sequencer_utils::deployer::is_proxy_contract;
 use serde::{Deserialize, Serialize};
+use url::Url;
 use vbs::version::Version;
 
 /// Initial configuration of an Espresso stake table.
@@ -85,8 +86,8 @@ impl Genesis {
 }
 
 impl Genesis {
-    pub async fn validate_fee_contract(&self, l1_rpc_url: String) -> anyhow::Result<()> {
-        let l1 = L1Client::new(l1_rpc_url.parse().context("invalid url")?);
+    pub async fn validate_fee_contract(&self, l1_rpc_url: Url) -> anyhow::Result<()> {
+        let l1 = L1Client::new(l1_rpc_url);
 
         if let Some(fee_contract_address) = self.chain_config.fee_contract {
             tracing::info!("validating fee contract at {fee_contract_address:x}");
@@ -593,7 +594,9 @@ mod test {
         let genesis: Genesis = toml::from_str(&toml).unwrap_or_else(|err| panic!("{err:#}"));
 
         // validate the fee_contract address
-        let result = genesis.validate_fee_contract(anvil.endpoint()).await;
+        let result = genesis
+            .validate_fee_contract(anvil.endpoint().parse().unwrap())
+            .await;
 
         // check if the result from the validation is an error
         if let Err(e) = result {
@@ -639,7 +642,9 @@ mod test {
         let genesis: Genesis = toml::from_str(&toml).unwrap_or_else(|err| panic!("{err:#}"));
 
         // Call the validation logic for the fee_contract address
-        let result = genesis.validate_fee_contract(anvil.endpoint()).await;
+        let result = genesis
+            .validate_fee_contract(anvil.endpoint().parse().unwrap())
+            .await;
 
         assert!(
             result.is_ok(),
@@ -711,7 +716,9 @@ mod test {
         let genesis: Genesis = toml::from_str(&toml).unwrap_or_else(|err| panic!("{err:#}"));
 
         // Call the validation logic for the fee_contract address
-        let result = genesis.validate_fee_contract(anvil.endpoint()).await;
+        let result = genesis
+            .validate_fee_contract(anvil.endpoint().parse().unwrap())
+            .await;
 
         assert!(
             result.is_ok(),
@@ -783,7 +790,9 @@ mod test {
         let genesis: Genesis = toml::from_str(&toml).unwrap_or_else(|err| panic!("{err:#}"));
 
         // Call the validation logic for the fee_contract address
-        let result = genesis.validate_fee_contract(anvil.endpoint()).await;
+        let result = genesis
+            .validate_fee_contract(anvil.endpoint().parse().unwrap())
+            .await;
 
         // check if the result from the validation is an error
         if let Err(e) = result {
@@ -851,7 +860,9 @@ mod test {
         let rpc_url = "https://ethereum-sepolia.publicnode.com";
 
         // validate the fee_contract address
-        let result = genesis.validate_fee_contract(rpc_url.to_string()).await;
+        let result = genesis
+            .validate_fee_contract(rpc_url.parse().unwrap())
+            .await;
 
         // check if the result from the validation is an error
         if let Err(e) = result {
@@ -904,7 +915,9 @@ mod test {
         let rpc_url = "https://ethereum-sepolia.publicnode.com";
 
         // validate the fee_contract address
-        let result = genesis.validate_fee_contract(rpc_url.to_string()).await;
+        let result = genesis
+            .validate_fee_contract(rpc_url.parse().unwrap())
+            .await;
 
         // check if the result from the validation is an error
         if let Err(e) = result {

diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs
@@ -185,7 +185,7 @@ pub struct NetworkParams {
 }
 
 pub struct L1Params {
-    pub url: Url,
+    pub urls: Vec<Url>,
     pub options: L1ClientOptions,
 }
 
@@ -491,7 +491,7 @@ pub async fn init_node<P: SequencerPersistence, V: Versions>(
     let l1_client = l1_params
         .options
         .with_metrics(metrics)
-        .connect(l1_params.url);
+        .connect(l1_params.urls);
     l1_client.spawn_tasks().await;
     let l1_genesis = match genesis.l1_finalized {
         L1Finalized::Block(b) => b,

diff --git a/sequencer/src/options.rs b/sequencer/src/options.rs
@@ -340,10 +340,12 @@ pub struct Options {
     #[clap(
         long,
         env = "ESPRESSO_SEQUENCER_L1_PROVIDER",
-        default_value = "http://localhost:8545"
+        default_value = "http://localhost:8545",
+        value_delimiter = ',',
+        num_args = 1..,
     )]
-    #[derivative(Debug(format_with = "Display::fmt"))]
-    pub l1_provider_url: Url,
+    #[derivative(Debug = "ignore")]
+    pub l1_provider_url: Vec<Url>,
 
     /// Configuration for the L1 client.
     #[clap(flatten)]

diff --git a/sequencer/src/run.rs b/sequencer/src/run.rs
@@ -28,7 +28,7 @@ pub async fn main() -> anyhow::Result<()> {
 
     // validate that the fee contract is a proxy and panic otherwise
     genesis
-        .validate_fee_contract(opt.l1_provider_url.to_string())
+        .validate_fee_contract(opt.l1_provider_url[0].clone())
         .await
         .unwrap();
 
@@ -130,7 +130,7 @@ where
 {
     let (private_staking_key, private_state_key) = opt.private_keys()?;
     let l1_params = L1Params {
-        url: opt.l1_provider_url,
+        urls: opt.l1_provider_url,
         options: opt.l1_options,
     };
 

diff --git a/types/Cargo.toml b/types/Cargo.toml
@@ -11,6 +11,7 @@ testing = []
 anyhow = { workspace = true }
 ark-serialize = { workspace = true }
 async-broadcast = { workspace = true }
+async-lock = { workspace = true }
 async-trait = { workspace = true }
 base64-bytes = { workspace = true }
 bincode = { workspace = true }
@@ -39,6 +40,7 @@ num-traits = { workspace = true }
 paste = { workspace = true }
 pretty_assertions = { workspace = true }
 rand = { workspace = true }
+reqwest = "0.11" # Same version used by ethers
 sequencer-utils = { path = "../utils" }
 serde = { workspace = true }
 serde_json = { workspace = true }