Add retries for safe commands (#12)

* Add retries for safe commands

* Update changelog

* Retries for all calls to o2_alert

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Next version
+
+### ✨ Improved
+
+* Add additional retries for valve and check safe commands.
+
+
 ## 0.3.5 - 2024-11-28
 
 ### ✨ Improved

src/lvmcryo/handlers/ln2.py

@@ -25,6 +25,7 @@
 
 from lvmopstools.devices.specs import spectrograph_pressures, spectrograph_temperatures
 from lvmopstools.devices.thermistors import read_thermistors
+from lvmopstools.retrier import Retrier
 from sdsstools.utils import GatheringTaskGroup
 
 from lvmcryo.config import ValveConfig, get_internal_config
@@ -182,6 +183,8 @@ async def check(
 
         """
 
+        retrier = Retrier(max_attempts=3, delay=1)
+
         # Check O2 alarms.
         if not self.alerts_route:
             self.log.warning("No alerts route provided. Not checking O2 alarms.")
@@ -207,7 +210,7 @@ async def check(
         if max_temperature is not None:
             self.log.info("Checking LN2 temperatures ...")
             try:
-                spec_temperatures = await spectrograph_temperatures()
+                spec_temperatures = await retrier(spectrograph_temperatures)()
             except Exception as err:
                 self.fail(f"Failed reading spectrograph temperatures: {err}")
             else:
@@ -227,7 +230,7 @@ async def check(
         if max_pressure is not None:
             self.log.info("Checking pressures ...")
             try:
-                spec_pressures = await spectrograph_pressures()
+                spec_pressures = await retrier(spectrograph_pressures)()
             except Exception as err:
                 self.fail(f"Failed reading spectrograph pressures: {err}")
             else:

src/lvmcryo/handlers/valve.py

@@ -18,18 +18,18 @@
 from rich.progress import TaskID
 
 from lvmopstools.clu import CluClient
+from lvmopstools.retrier import Retrier
 
 from lvmcryo.config import get_internal_config
 from lvmcryo.handlers.thermistor import ThermistorHandler
-from lvmcryo.tools import (
-    cancel_task,
-    get_fake_logger,
-)
+from lvmcryo.tools import cancel_task, get_fake_logger
 
 
 if TYPE_CHECKING:
     from sdsstools.configuration import Configuration
 
+    from lvmcryo.tools import TimerProgressBar
+
 
 __all__ = [
     "ValveHandler",
@@ -40,6 +40,7 @@
 ]
 
 
+@Retrier(max_attempts=3, delay=1)
 async def outlet_info(actor: str, outlet: str) -> dict[str, Any]:
     """Retrieves outlet information from the NPS."""
 
@@ -51,6 +52,7 @@ async def outlet_info(actor: str, outlet: str) -> dict[str, Any]:
     return cmd.replies.get("outlet_info")
 
 
+@Retrier(max_attempts=3, delay=1)
 async def valve_on_off(
     actor: str,
     outlet_name: str,
@@ -118,6 +120,7 @@ async def valve_on_off(
     return
 
 
+@Retrier(max_attempts=3, delay=1)
 async def cancel_nps_threads(actor: str, thread_id: int | None = None):
     """Cancels a script thread in an NPS.
 
@@ -136,6 +139,7 @@ async def cancel_nps_threads(actor: str, thread_id: int | None = None):
         await client.send_command(actor, command_string)
 
 
+@Retrier(max_attempts=3, delay=1)
 async def close_all_valves(config: Configuration | None = None, dry_run: bool = False):
     """Closes all the outlets."""
 
@@ -155,10 +159,6 @@ async def close_all_valves(config: Configuration | None = None, dry_run: bool =
     )
 
 
-if TYPE_CHECKING:
-    from lvmcryo.tools import TimerProgressBar
-
-
 @dataclass
 class ValveHandler:
     """Handles a valve, including opening and closing, timeouts, and thermistors.

src/lvmcryo/tools.py

@@ -26,6 +26,7 @@
 from rich.console import Console
 from rich.progress import BarColumn, MofNCompleteColumn, Progress, TaskID, TextColumn
 
+from lvmopstools.retrier import Retrier
 from sdsstools.logger import CustomJsonFormatter
 from sdsstools.utils import run_in_executor
 
@@ -244,6 +245,7 @@ def get_fake_logger():
     return logger
 
 
+@Retrier(max_attempts=3, delay=0.5)
 async def o2_alert(route: str = "http://lvm-hub.lco.cl:8090/api/alerts"):
     """Is there an active O2 alert?"""