diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1e303..29afb3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Next version + +### ✨ Improved + +* Add additional retries for valve and check safe commands. + + ## 0.3.5 - 2024-11-28 ### ✨ Improved diff --git a/src/lvmcryo/handlers/ln2.py b/src/lvmcryo/handlers/ln2.py index 9e2e0e9..9100c47 100644 --- a/src/lvmcryo/handlers/ln2.py +++ b/src/lvmcryo/handlers/ln2.py @@ -25,6 +25,7 @@ from lvmopstools.devices.specs import spectrograph_pressures, spectrograph_temperatures from lvmopstools.devices.thermistors import read_thermistors +from lvmopstools.retrier import Retrier from sdsstools.utils import GatheringTaskGroup from lvmcryo.config import ValveConfig, get_internal_config @@ -182,6 +183,8 @@ async def check( """ + retrier = Retrier(max_attempts=3, delay=1) + # Check O2 alarms. if not self.alerts_route: self.log.warning("No alerts route provided. Not checking O2 alarms.") @@ -207,7 +210,7 @@ async def check( if max_temperature is not None: self.log.info("Checking LN2 temperatures ...") try: - spec_temperatures = await spectrograph_temperatures() + spec_temperatures = await retrier(spectrograph_temperatures)() except Exception as err: self.fail(f"Failed reading spectrograph temperatures: {err}") else: @@ -227,7 +230,7 @@ async def check( if max_pressure is not None: self.log.info("Checking pressures ...") try: - spec_pressures = await spectrograph_pressures() + spec_pressures = await retrier(spectrograph_pressures)() except Exception as err: self.fail(f"Failed reading spectrograph pressures: {err}") else: diff --git a/src/lvmcryo/handlers/valve.py b/src/lvmcryo/handlers/valve.py index 31636b1..7bc61c8 100644 --- a/src/lvmcryo/handlers/valve.py +++ b/src/lvmcryo/handlers/valve.py @@ -18,18 +18,18 @@ from rich.progress import TaskID from lvmopstools.clu import CluClient +from lvmopstools.retrier import Retrier from lvmcryo.config import get_internal_config from lvmcryo.handlers.thermistor import ThermistorHandler -from lvmcryo.tools import ( - cancel_task, - get_fake_logger, -) +from lvmcryo.tools import cancel_task, get_fake_logger if TYPE_CHECKING: from sdsstools.configuration import Configuration + from lvmcryo.tools import TimerProgressBar + __all__ = [ "ValveHandler", @@ -40,6 +40,7 @@ ] +@Retrier(max_attempts=3, delay=1) async def outlet_info(actor: str, outlet: str) -> dict[str, Any]: """Retrieves outlet information from the NPS.""" @@ -51,6 +52,7 @@ async def outlet_info(actor: str, outlet: str) -> dict[str, Any]: return cmd.replies.get("outlet_info") +@Retrier(max_attempts=3, delay=1) async def valve_on_off( actor: str, outlet_name: str, @@ -118,6 +120,7 @@ async def valve_on_off( return +@Retrier(max_attempts=3, delay=1) async def cancel_nps_threads(actor: str, thread_id: int | None = None): """Cancels a script thread in an NPS. @@ -136,6 +139,7 @@ async def cancel_nps_threads(actor: str, thread_id: int | None = None): await client.send_command(actor, command_string) +@Retrier(max_attempts=3, delay=1) async def close_all_valves(config: Configuration | None = None, dry_run: bool = False): """Closes all the outlets.""" @@ -155,10 +159,6 @@ async def close_all_valves(config: Configuration | None = None, dry_run: bool = ) -if TYPE_CHECKING: - from lvmcryo.tools import TimerProgressBar - - @dataclass class ValveHandler: """Handles a valve, including opening and closing, timeouts, and thermistors. diff --git a/src/lvmcryo/tools.py b/src/lvmcryo/tools.py index e03295a..db6a877 100644 --- a/src/lvmcryo/tools.py +++ b/src/lvmcryo/tools.py @@ -26,6 +26,7 @@ from rich.console import Console from rich.progress import BarColumn, MofNCompleteColumn, Progress, TaskID, TextColumn +from lvmopstools.retrier import Retrier from sdsstools.logger import CustomJsonFormatter from sdsstools.utils import run_in_executor @@ -244,6 +245,7 @@ def get_fake_logger(): return logger +@Retrier(max_attempts=3, delay=0.5) async def o2_alert(route: str = "http://lvm-hub.lco.cl:8090/api/alerts"): """Is there an active O2 alert?"""