From c11d285a1a2f38186cff72c6e09731ecbc881de0 Mon Sep 17 00:00:00 2001 From: popcnt <142196625+popcnt1@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:55:22 +0800 Subject: [PATCH] chore(da): improve error logging for segment submission (#3015) * fix(backend): improve error logging for segment submission Enhanced error logging in Celestia and Avail backends to include attempts count and error/status details during segment submission failures. --- crates/rooch-da/src/backend/openda/avail.rs | 25 ++++++++++--------- crates/rooch-da/src/backend/openda/backend.rs | 2 +- .../rooch-da/src/backend/openda/celestia.rs | 23 +++++++++-------- crates/rooch-da/src/backend/openda/mod.rs | 2 +- .../src/backend/openda/{fs.rs => opendal.rs} | 0 5 files changed, 27 insertions(+), 25 deletions(-) rename crates/rooch-da/src/backend/openda/{fs.rs => opendal.rs} (100%) diff --git a/crates/rooch-da/src/backend/openda/avail.rs b/crates/rooch-da/src/backend/openda/avail.rs index 1469267904..533e0338ed 100644 --- a/crates/rooch-da/src/backend/openda/avail.rs +++ b/crates/rooch-da/src/backend/openda/avail.rs @@ -17,6 +17,7 @@ pub(crate) const DEFAULT_AVAIL_MAX_SEGMENT_SIZE: u64 = 256 * 1024; const BACK_OFF_MIN_DELAY: Duration = Duration::from_millis(3000); const SUBMIT_API_PATH: &str = "v2/submit"; +#[derive(Clone)] pub(crate) struct AvailClient { endpoint: String, client: Client, @@ -53,11 +54,12 @@ impl Operator for AvailClient { ) -> anyhow::Result<()> { let submit_url = format!("{}/{}", self.endpoint, SUBMIT_API_PATH); let data = general_purpose::STANDARD.encode(&segment_bytes); - let max_retries = self.max_retries; - let mut retries = 0; + let max_attempts = self.max_retries + 1; // max_attempts = max_retries + first attempt + let mut attempts = 0; let mut retry_delay = BACK_OFF_MIN_DELAY; loop { + attempts += 1; let response = self .client .post(&submit_url) @@ -65,7 +67,6 @@ impl Operator for AvailClient { .body(json!({ "data": data }).to_string()) .send() .await?; - match response.status() { StatusCode::OK => { let submit_response: AvailSubmitResponse = response.json().await?; @@ -85,22 +86,22 @@ impl Operator for AvailClient { )) } _ => { - if retries <= max_retries { - retries += 1; - sleep(retry_delay).await; - retry_delay *= 3; // Exponential backoff + if attempts < max_attempts { tracing::warn!( - "Failed to submit segment: {:?} to Avail, retrying after {}ms, attempt: {}", + "Failed to submit segment: {:?} to Avail: {}, attempts: {}，retrying after {}ms", segment_id, + response.status(), + attempts, retry_delay.as_millis(), - retries ); + sleep(retry_delay).await; + retry_delay *= 3; // Exponential backoff } else { return Err(anyhow!( - "Failed to submit segment: {:?} to Avail after {} attempts: {}", + "Failed to submit segment: {:?} to Avail: {} after {} attempts", segment_id, - retries, - response.status() + response.status(), + attempts, )); } } diff --git a/crates/rooch-da/src/backend/openda/backend.rs b/crates/rooch-da/src/backend/openda/backend.rs index e15b68be35..a4e3bb750b 100644 --- a/crates/rooch-da/src/backend/openda/backend.rs +++ b/crates/rooch-da/src/backend/openda/backend.rs @@ -3,7 +3,7 @@ use crate::backend::openda::avail::AvailClient; use crate::backend::openda::celestia::{CelestiaClient, WrappedNamespace}; -use crate::backend::openda::fs::BACK_OFF_MIN_DELAY; +use crate::backend::openda::opendal::BACK_OFF_MIN_DELAY; use crate::backend::openda::operator::{Operator, OperatorConfig}; use crate::backend::DABackend; use async_trait::async_trait; diff --git a/crates/rooch-da/src/backend/openda/celestia.rs b/crates/rooch-da/src/backend/openda/celestia.rs index 75f89b0496..1921fdf3ec 100644 --- a/crates/rooch-da/src/backend/openda/celestia.rs +++ b/crates/rooch-da/src/backend/openda/celestia.rs @@ -52,11 +52,12 @@ impl Operator for CelestiaClient { _prefix: Option, ) -> anyhow::Result<()> { let blob = Blob::new(self.namespace, segment_bytes)?; - let max_retries = self.max_retries; - let mut retries = 0; + let max_attempts = self.max_retries + 1; // max_attempts = max_retries + first attempt + let mut attempts = 0; let mut retry_delay = BACK_OFF_MIN_DELAY; loop { + attempts += 1; match self .client .blob_submit(&[blob.clone()], SubmitOptions::default()) @@ -72,22 +73,22 @@ impl Operator for CelestiaClient { return Ok(()); } Err(e) => { - if retries <= max_retries { - retries += 1; - sleep(retry_delay).await; - retry_delay *= 3; + if attempts < max_attempts { tracing::warn!( - "Failed to submit segment: {:?} to Celestia, retrying after {}ms, attempt: {}", + "Failed to submit segment: {:?} to Celestia: {:?}, attempts: {}, retrying after {}ms", segment_id, + e, + attempts, retry_delay.as_millis(), - retries ); + sleep(retry_delay).await; + retry_delay *= 3; } else { return Err(anyhow!( - "Failed to submit segment: {:?} to Celestia after {} attempts: {:?}", + "Failed to submit segment: {:?} to Celestia: {:?} after {} attempts", segment_id, - retries, - e + e, + attempts, )); } } diff --git a/crates/rooch-da/src/backend/openda/mod.rs b/crates/rooch-da/src/backend/openda/mod.rs index 4261f2cf72..4c91cf8afc 100644 --- a/crates/rooch-da/src/backend/openda/mod.rs +++ b/crates/rooch-da/src/backend/openda/mod.rs @@ -4,7 +4,7 @@ mod avail; mod backend; mod celestia; -mod fs; +mod opendal; mod operator; pub use self::backend::OpenDABackend; diff --git a/crates/rooch-da/src/backend/openda/fs.rs b/crates/rooch-da/src/backend/openda/opendal.rs similarity index 100% rename from crates/rooch-da/src/backend/openda/fs.rs rename to crates/rooch-da/src/backend/openda/opendal.rs