Skip to content

Commit

Permalink
Better logs. (#287)
Browse files Browse the repository at this point in the history
slot timings

## πŸ“ Summary

- Timing info about "Received payload"
- Watchdog kill reason.

## πŸ’‘ Motivation and Context

Help problem identification on prod.

## βœ… I have completed the following steps:

* [X] Run `make lint`
* [X] Run `make test`
* [ ] Added tests (if applicable)
  • Loading branch information
ZanCorDX authored Dec 13, 2024
1 parent 9835e67 commit 171d8fc
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 5 deletions.
10 changes: 8 additions & 2 deletions crates/rbuilder/src/live_builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,10 @@ where
);

let watchdog_sender = match self.watchdog_timeout {
Some(duration) => Some(spawn_watchdog_thread(duration)?),
Some(duration) => Some(spawn_watchdog_thread(
duration,
"block build started".to_string(),
)?),
None => {
info!("Watchdog not enabled");
None
Expand All @@ -203,11 +206,14 @@ where
}) {
continue;
}
let current_time = OffsetDateTime::now_utc();
// see if we can get parent header in a reasonable time
let time_to_slot = payload.timestamp() - OffsetDateTime::now_utc();
let time_to_slot = payload.timestamp() - current_time;
debug!(
slot = payload.slot(),
block = payload.block(),
?current_time,
payload_timestamp = ?payload.timestamp(),
?time_to_slot,
"Received payload, time till slot timestamp",
);
Expand Down
10 changes: 7 additions & 3 deletions crates/rbuilder/src/live_builder/watchdog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ use tracing::{error, info};

/// Spawns a thread that will kill the process if there is no events sent on the channel
/// for the timeout time.
pub fn spawn_watchdog_thread(timeout: Duration) -> io::Result<flume::Sender<()>> {
/// context is a string to be logged to be able to distinguish different types of deaths.
pub fn spawn_watchdog_thread(timeout: Duration, context: String) -> io::Result<flume::Sender<()>> {
let (sender, receiver) = flume::unbounded();
std::thread::Builder::new()
.name(String::from("watchdog"))
Expand All @@ -13,15 +14,18 @@ pub fn spawn_watchdog_thread(timeout: Duration) -> io::Result<flume::Sender<()>>
match receiver.recv_timeout(timeout) {
Ok(()) => {}
Err(RecvTimeoutError::Timeout) => {
error!("Watchdog timeout");
error!(context, "Watchdog timeout");
std::process::exit(1);
}
Err(RecvTimeoutError::Disconnected) => {
break;
}
}
}
info!("Watchdog finished, will kill application in 12 seconds");
info!(
context,
"Watchdog finished, will kill application in 12 seconds"
);

std::thread::sleep(Duration::from_secs(12));
std::process::exit(1);
Expand Down

0 comments on commit 171d8fc

Please sign in to comment.