-
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
light-client: Move backoff logic from animator to light client
- Loading branch information
Showing
7 changed files
with
141 additions
and
58 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| use std::time::Duration as StdDuration; | ||
|
|
||
| use async_trait::async_trait; | ||
| use chrono::{DateTime, Duration, Utc}; | ||
| use log::{info, warn}; | ||
| use tokio::sync::Mutex; | ||
|
|
||
| use crate::{LightClient, LightClientError}; | ||
|
|
||
| #[derive(PartialEq)] | ||
| enum ConnectionStatus { | ||
| Healthy, | ||
| IntermittentFailure, | ||
| ProlongedFailure, | ||
| } | ||
|
|
||
| struct BackoffState { | ||
| status: ConnectionStatus, | ||
| delay: Duration, | ||
| next_check: DateTime<Utc>, | ||
| } | ||
|
|
||
| pub struct BackoffDecorator<T: LightClient> { | ||
| inner: T, | ||
| start_delay: Duration, | ||
| max_delay: Duration, | ||
| timeout: StdDuration, | ||
| state: Mutex<BackoffState>, | ||
| } | ||
|
|
||
| impl<T: LightClient> BackoffDecorator<T> { | ||
| pub fn new(light_client: T) -> BackoffDecorator<T> { | ||
| let default_start_delay = Duration::seconds(1); | ||
| Self { | ||
| inner: light_client, | ||
| start_delay: default_start_delay, | ||
| max_delay: Duration::seconds(8), | ||
| timeout: StdDuration::from_millis(100), | ||
| state: Mutex::new(BackoffState { | ||
| status: ConnectionStatus::Healthy, | ||
| delay: default_start_delay, | ||
| next_check: Utc::now(), | ||
| }), | ||
| } | ||
| } | ||
|
|
||
| pub fn with_start_delay(mut self, delay: Duration) -> Self { | ||
| self.start_delay = delay; | ||
| self | ||
| } | ||
|
|
||
| pub fn with_max_delay(mut self, delay: Duration) -> Self { | ||
| self.max_delay = delay; | ||
| self | ||
| } | ||
|
|
||
| pub fn with_timeout(mut self, timeout: StdDuration) -> Self { | ||
| self.timeout = timeout; | ||
| self | ||
| } | ||
| } | ||
|
|
||
| #[async_trait] | ||
| impl<T> LightClient for BackoffDecorator<T> | ||
| where | ||
| T: LightClient + Send + Sync, | ||
| { | ||
| async fn display_frame(&self, frame: &lightfx::Frame) -> Result<(), LightClientError> { | ||
| let mut state = self.state.lock().await; | ||
| let now = Utc::now(); | ||
| if now < state.next_check && state.status != ConnectionStatus::Healthy { | ||
| return Err(LightClientError::ConnectionLost); | ||
| } | ||
|
|
||
| match tokio::time::timeout(self.timeout, self.inner.display_frame(frame)).await { | ||
| Ok(Ok(_)) => { | ||
| if state.status != ConnectionStatus::Healthy { | ||
| info!("Regained connection to light client"); | ||
| } | ||
| state.status = ConnectionStatus::Healthy; | ||
| state.delay = self.start_delay; | ||
| state.next_check = now; | ||
| Ok(()) | ||
| } | ||
| Ok(Err(LightClientError::ConnectionLost)) | Err(_) => { | ||
| state.next_check = now + state.delay; | ||
| if state.delay < self.max_delay { | ||
| state.status = ConnectionStatus::IntermittentFailure; | ||
| warn!( | ||
| "Failed to send frame to remote lights, will retry in {:.2} seconds", | ||
| state.delay.num_milliseconds() as f64 / 1000.0 | ||
| ); | ||
| } else if state.status != ConnectionStatus::ProlongedFailure { | ||
| state.status = ConnectionStatus::ProlongedFailure; | ||
| warn!( | ||
| "Lost connection to lights, will continue retrying every {:.2} seconds", | ||
| self.max_delay.num_milliseconds() as f64 / 1000.0 | ||
| ); | ||
| } | ||
| state.delay = (state.delay * 2).min(self.max_delay); | ||
| Err(LightClientError::ConnectionLost) | ||
| } | ||
| Ok(Err(LightClientError::ProcessExited)) => { | ||
| warn!("Light client exited, exiting"); | ||
| Err(LightClientError::ProcessExited) | ||
| } | ||
| _ => Err(LightClientError::Unlikely), | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| pub mod backoff_decorator; | ||
| pub mod combined; | ||
| pub mod feedback; | ||
| pub mod http; | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters