Skip to content

Commit

Permalink
chore(da): improve error logging for segment submission (#3015)
Browse files Browse the repository at this point in the history
* fix(backend): improve error logging for segment submission

Enhanced error logging in Celestia and Avail backends to include attempts count and error/status details during segment submission failures.
  • Loading branch information
popcnt1 authored Dec 5, 2024
1 parent 2ef805b commit c11d285
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 25 deletions.
25 changes: 13 additions & 12 deletions crates/rooch-da/src/backend/openda/avail.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pub(crate) const DEFAULT_AVAIL_MAX_SEGMENT_SIZE: u64 = 256 * 1024;
const BACK_OFF_MIN_DELAY: Duration = Duration::from_millis(3000);
const SUBMIT_API_PATH: &str = "v2/submit";

#[derive(Clone)]
pub(crate) struct AvailClient {
endpoint: String,
client: Client,
Expand Down Expand Up @@ -53,19 +54,19 @@ impl Operator for AvailClient {
) -> anyhow::Result<()> {
let submit_url = format!("{}/{}", self.endpoint, SUBMIT_API_PATH);
let data = general_purpose::STANDARD.encode(&segment_bytes);
let max_retries = self.max_retries;
let mut retries = 0;
let max_attempts = self.max_retries + 1; // max_attempts = max_retries + first attempt
let mut attempts = 0;
let mut retry_delay = BACK_OFF_MIN_DELAY;

loop {
attempts += 1;
let response = self
.client
.post(&submit_url)
.header("Content-Type", "application/json")
.body(json!({ "data": data }).to_string())
.send()
.await?;

match response.status() {
StatusCode::OK => {
let submit_response: AvailSubmitResponse = response.json().await?;
Expand All @@ -85,22 +86,22 @@ impl Operator for AvailClient {
))
}
_ => {
if retries <= max_retries {
retries += 1;
sleep(retry_delay).await;
retry_delay *= 3; // Exponential backoff
if attempts < max_attempts {
tracing::warn!(
"Failed to submit segment: {:?} to Avail, retrying after {}ms, attempt: {}",
"Failed to submit segment: {:?} to Avail: {}, attempts: {},retrying after {}ms",
segment_id,
response.status(),
attempts,
retry_delay.as_millis(),
retries
);
sleep(retry_delay).await;
retry_delay *= 3; // Exponential backoff
} else {
return Err(anyhow!(
"Failed to submit segment: {:?} to Avail after {} attempts: {}",
"Failed to submit segment: {:?} to Avail: {} after {} attempts",
segment_id,
retries,
response.status()
response.status(),
attempts,
));
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/rooch-da/src/backend/openda/backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

use crate::backend::openda::avail::AvailClient;
use crate::backend::openda::celestia::{CelestiaClient, WrappedNamespace};
use crate::backend::openda::fs::BACK_OFF_MIN_DELAY;
use crate::backend::openda::opendal::BACK_OFF_MIN_DELAY;
use crate::backend::openda::operator::{Operator, OperatorConfig};
use crate::backend::DABackend;
use async_trait::async_trait;
Expand Down
23 changes: 12 additions & 11 deletions crates/rooch-da/src/backend/openda/celestia.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,12 @@ impl Operator for CelestiaClient {
_prefix: Option<String>,
) -> anyhow::Result<()> {
let blob = Blob::new(self.namespace, segment_bytes)?;
let max_retries = self.max_retries;
let mut retries = 0;
let max_attempts = self.max_retries + 1; // max_attempts = max_retries + first attempt
let mut attempts = 0;
let mut retry_delay = BACK_OFF_MIN_DELAY;

loop {
attempts += 1;
match self
.client
.blob_submit(&[blob.clone()], SubmitOptions::default())
Expand All @@ -72,22 +73,22 @@ impl Operator for CelestiaClient {
return Ok(());
}
Err(e) => {
if retries <= max_retries {
retries += 1;
sleep(retry_delay).await;
retry_delay *= 3;
if attempts < max_attempts {
tracing::warn!(
"Failed to submit segment: {:?} to Celestia, retrying after {}ms, attempt: {}",
"Failed to submit segment: {:?} to Celestia: {:?}, attempts: {}, retrying after {}ms",
segment_id,
e,
attempts,
retry_delay.as_millis(),
retries
);
sleep(retry_delay).await;
retry_delay *= 3;
} else {
return Err(anyhow!(
"Failed to submit segment: {:?} to Celestia after {} attempts: {:?}",
"Failed to submit segment: {:?} to Celestia: {:?} after {} attempts",
segment_id,
retries,
e
e,
attempts,
));
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/rooch-da/src/backend/openda/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
mod avail;
mod backend;
mod celestia;
mod fs;
mod opendal;
mod operator;

pub use self::backend::OpenDABackend;
File renamed without changes.

0 comments on commit c11d285

Please sign in to comment.