Added floco lock and cs8recovery procedure

daq_lib.py

@@ -16,7 +16,7 @@
 from daq_utils import getBlConfig
 from config_params import *
 from kafka_producer import send_kafka_message
-from start_bs import govs, gov_robot, flyer, RE
+from start_bs import govs, gov_robot, flyer, RE, gonio, robot_arm
 import gov_lib
 from bluesky.preprocessors import finalize_wrapper
 import bluesky.plan_stubs as bps
@@ -28,7 +28,11 @@
   import ispybLib
 except Exception as e:
   logger.error("daq_lib: ISPYB import error, %s" % e)
-
+
+if daq_utils.beamline in ["amx", "fmx"]:
+  from start_bs import gov_mon_signal
+
+
 #all keys below are created as beamlineComm PVs. if adding items here, be sure to add them to the simulator
 var_list = {'beam_check_flag':0,'overwrite_check_flag':1,'omega':0.00,'kappa':0.00,'phi':0.00,'theta':0.00,'distance':10.00,'rot_dist0':300.0,'inc0':1.00,'exptime0':5.00,'file_prefix0':'lowercase','numstart0':0,'col_start0':0.00,'col_end0':1.00,'scan_axis':'omega','wavelength0':1.1,'datum_omega':0.00,'datum_kappa':0.00,'datum_phi':0.00,'size_mode':0,'spcgrp':1,'state':"Idle",'state_percent':0,'datafilename':'none','active_sweep':-1,'html_logging':1,'take_xtal_pics':0,'px_id':'none','xtal_id':'none','current_pinpos':0,'sweep_count':0,'group_name':'none','mono_energy_target':1.1,'mono_wave_target':1.1,'energy_inflection':12398.5,'energy_peak':12398.5,'wave_inflection':1.0,'wave_peak':1.0,'energy_fall':12398.5,'wave_fall':1.0,'beamline_merit':0,'fprime_peak':0.0,'f2prime_peak':0.0,'fprime_infl':0.0,'f2prime_infl':0.0,'program_state':"Program Ready",'filter':0,'edna_aimed_completeness':0.99,'edna_aimed_ISig':2.0,'edna_aimed_multiplicity':'auto','edna_aimed_resolution':'auto','mono_energy_current':1.1,'mono_energy_scan_step':1,'mono_wave_current':1.1,'mono_scan_points':21,'mounted_pin':(db_lib.beamlineInfo(daq_utils.beamline, 'mountedSample')["sampleID"]),'pause_button_state':'Pause','vector_on':0,'vector_fpp':1,'vector_step':0.0,'vector_translation':0.0,'xia2_on':0,'grid_exptime':0.2,'grid_imwidth':0.2,'choochResultFlag':"0",'xrecRasterFlag':"0"}
 
@@ -218,6 +222,42 @@ def unlockGUI():
 def lockGUI():
   logger.info('locking GUI')
   beamline_support.set_any_epics_pv(daq_utils.beamlineComm+"zinger_flag","VAL",-1)  
+
+def govMonOn():
+  gov_mon_signal.set(1)
+
+def govMonOff():
+  gov_mon_signal.set(0)
+
+def flocoLock():
+  lockGUI()
+  govMonOff()
+
+def flocoUnlock():
+  unlockGUI()
+  govMonOn()
+
+def flocoStopOperations():
+  try:
+    daq_macros.run_recovery_procedure(stop=True)
+    lockGUI()
+  except Exception as e:
+    logger.exception("Error encountered while running flocoStopOperations. Stopping...")
+
+def flocoContinueOperations():
+  try:
+    daq_macros.run_recovery_procedure(stop=False)
+    flocoUnlock()
+    beamline_support.set_any_epics_pv(daq_utils.beamlineComm + "command_s", "VAL", 
+      json.dumps(
+              {
+                  "function": "runDCQueue",
+                  "args": [],
+                  "kwargs": {},
+              }
+          ))
+  except Exception as e:
+    logger.exception("Error encountered while running flocoContinueOperations. Stopping...")
 
 def refreshGuiTree():
   beamline_support.set_any_epics_pv(daq_utils.beamlineComm+"live_q_change_flag","VAL",1)
@@ -384,14 +424,25 @@ def waitBeam():
         gui_message("Waiting for beam. Type beamCheckOff() in lsdcServer window to continue.")
       time.sleep(1.0)
 
+def waitRobotArm():
+  waiting = True
+  while waiting:
+    gui_message("Robot arm speed not at 100%. Set speed to 100")
+    time.sleep(1.0)
+    if robot_arm.is_full_speed():
+      waiting = False
+
 def runDCQueue(): #maybe don't run rasters from here???
   global abort_flag
 
   autoMounted = 0 #this means the mount was performed from a runQueue, as opposed to a manual mount button push
   logger.info("running queue in daq server")
   while (1):
-    if (getBlConfig("queueCollect") == 1 and getBlConfig(BEAM_CHECK) == 1):
-      waitBeam()
+    if (getBlConfig("queueCollect") == 1): 
+      if (getBlConfig(BEAM_CHECK) == 1):
+        waitBeam()
+      if not start_bs.robot_arm.is_full_speed():
+        waitRobotArm()
     if (abort_flag):
       abort_flag =  0 #careful about when to reset this
       return

daq_macros.py

@@ -259,11 +259,86 @@ def run_robot_recovery_procedure():
   """ Generic recovery procedure to be used during automated
   collection"""
   # Recover robot
+  logging.info("Running robot recovery procedure (nuclear option)")
+  logger.info("Recovering robot")
   robot_lib.recoverRobot()
   # Dry Gripper
+  logger.info("Drying gripper")
+  robot_lib.dryGripper()
+  # Dry Gripper
+  logger.info("Drying gripper, again")
   robot_lib.dryGripper()
   # Park Gripper and cool gripper
+  logger.info("Cooling gripper")
   robot_lib.cooldownGripper()
+  logger.info("Homing pins")
+  from start_bs import home_pins
+  RE(home_pins())
+  logger.info("Setting robot state to SE")
+  gov_lib.setGovRobot(gov_robot, 'SE')
+  logger.info("Turning govmon ON")
+  daq_lib.govMonOn()
+
+def recoverCS8():
+  logger.info("Starting CS8 recovery")
+  logger.info("Recovering robot")
+  robot_lib.recoverRobot()
+  logger.info("Drying gripper")
+  robot_lib.dryGripper()
+  rotCP = beamline_lib.motorPosFromDescriptor("dewarRot")
+  logger.info(f"Rotate dewar +1 degree, from {rotCP} to {rotCP+1}")
+  beamline_lib.mvaDescriptor("dewarRot",rotCP + 1)
+  logger.info(f"Rotate dewar -1 degree, from {rotCP+1} to {rotCP}")
+  beamline_lib.mvaDescriptor("dewarRot",rotCP)
+  logger.info("Setting robot state to SE")
+  gov_lib.setGovRobot(gov_robot, 'SE')
+  logger.info("Setting robot state to M")
+  gov_lib.setGovRobot(gov_robot, 'M')
+  logger.info("Setting robot state to SE")
+  gov_lib.setGovRobot(gov_robot, 'SE')
+
+def run_recovery_procedure(stop=True):
+  """
+  Manual recovery procedure used in daq_lib.flocoStopOperations and daq_lib.flocoContinueOperations
+  """
+  logger.info(f"Running recovery procedure: {'stop operations' if stop else 'continue operations'}")
+  def check_robot():
+    if not getBlConfig("robot_online") or not getBlConfig("mountEnabled"):
+      raise ValueError("Robot is offline or mount is disabled, sample found in gripper. Stopping recovery...")
+
+  def run_home_pin():
+    from start_bs import home_pins
+    RE(home_pins())
+
+  def gov_state_to_se():
+    gov_lib.setGovRobot(gov_robot, 'SE')
+
+  def check_robot_speed():
+    if not robot_arm.is_full_speed():
+      logger.error("Robot arm speed is NOT 100%")
+
+  def check_beam():
+    if not (getPvDesc("beamAvailable") or getBlConfig(BEAM_CHECK) == 0):
+      logger.error("Beam not available, please open shutter")
+
+  steps_to_run = {"Recover robot": robot_lib.recoverRobot, 
+                  "Check robot status": check_robot, 
+                  "Dry gripper": robot_lib.dryGripper, 
+                  "Home pin": run_home_pin, }
+  if stop:
+    steps_to_run.update({"Disable mount": disableMount, "Robot off": robotOff})
+  else:
+    steps_to_run.update({"Enable mount": enableMount, "Robot on": robotOn})
+
+  steps_to_run.update({"Move Governor to SE": gov_state_to_se, 
+                       "Checking robot arm speed": check_robot_speed,
+                       "Checking beam": check_beam})
+
+  for i, (step_message, func) in enumerate(steps_to_run.items()):
+    logger.info(f"Step {i+1} of {len(steps_to_run)}: {step_message}")
+    func()
+    logger.info(f"Completed step: {step_message}")
+
 
 def run_top_view_optimized():
     RE(topview_optimized())

embl_robot.py

@@ -275,9 +275,20 @@ def mount(self, gov_robot, puckPos,pinPos,sampID,**kwargs):
               except Exception as e:
                 e_s = str(e)
                 message = "ROBOT mount ERROR: " + e_s
-                daq_lib.gui_message(message)
-                logger.error(message)
-                return MOUNT_FAILURE
+                if "Load ABORT with exception Timeout" in e_s:
+                  daq_macros.run_robot_recovery_procedure()
+                  try:
+                    RobotControlLib.mount(absPos)
+                  except Exception as e:
+                    e_s = str(e)
+                    message = "ROBOT mount ERROR: " + e_s                    
+                    daq_lib.gui_message(message)
+                    logger.error(message)
+                    return MOUNT_FAILURE
+                else:
+                  daq_lib.gui_message(message)
+                  logger.error(message)
+                  return MOUNT_FAILURE
             else:
               time.sleep(0.5)
               if (getPvDesc("sampleDetected") == 0):

mxbluesky/devices/auto_center.py

@@ -4,6 +4,7 @@
 from ophyd import (DeviceStatus, Component as Cpt, Signal, EpicsSignal, Device, 
                     DDC_EpicsSignal, DDC_EpicsSignalRO, ADComponent as ADCpt, ImagePlugin,
                       ROIPlugin, TransformPlugin, StatsPlugin, ProcessPlugin, SingleTrigger, ProsilicaDetector)
+from ophyd.utils.errors import WaitTimeoutError
 from pathlib import Path
 import requests
 from bluesky.utils import FailedStatus
@@ -210,7 +211,7 @@ def trigger(self):
         try:
             status = super().trigger()
             status.wait(6)
-        except AttributeError:
+        except (AttributeError, WaitTimeoutError) as e:
             raise FailedStatus
         return status
 

mxbluesky/devices/auto_recovery.py

@@ -0,0 +1,34 @@
+from ophyd import Device, EpicsSignalRO, EpicsSignal, Component as Cpt
+from ophyd.status import SubscriptionStatus
+
+
+class PYZHomer(Device):
+
+    status = Cpt(EpicsSignalRO, "XF:17IDB-ES:AMX{Sentinel}Homing_Sts")
+    home_actuate = Cpt(EpicsSignal, "XF:17ID:AMX{Sentinel}pin_home")
+
+    kill_home = Cpt(EpicsSignal, "XF:17IDB-ES:AMX{Sentinel}Homing_Kill")
+    kill_py = Cpt(EpicsSignal, "XF:17IDB-ES:AMX{Gon:1-Ax:PY}Cmd:Kill-Cmd")
+    kill_pz = Cpt(EpicsSignal, "XF:17IDB-ES:AMX{Gon:1-Ax:PZ}Cmd:Kill-Cmd")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def trigger(self):
+
+        def callback_homed(value, old_value, **kwargs):
+            if old_value == 1 and value == 0:
+                return True
+            else:
+                return False
+
+        self.home_actuate.put(1)
+
+        homing_status = SubscriptionStatus(
+            self.status,
+            callback_homed,
+            run=False,
+            timeout=180,
+        )
+
+        return homing_status

mxbluesky/devices/generic.py

@@ -33,4 +33,45 @@ def __init__(self, *args, **kwargs):
         self.cy = self.py
         self.z = self.pz
         self.cz = self.pz
-        self.omega = self.o
+        self.omega = self.o
+
+
+class Dewar(Device):
+    rotation = Cpt(EpicsSignal, "{Dew:1-Ax:R}Virtual")
+    rotation_motor = Cpt(EpicsMotor, "{Dew:1-Ax:R}Mtr")
+
+    def rotate(self, rotation_angle, absolute=True):
+        def check_value_sink(*, old_value, value, **kwargs):
+            "Return True when the movement is complete, False otherwise."
+            return old_value == 1 and value == 0
+
+        def check_value_raise(*, old_value, value, **kwargs):
+            "Return True when the movement is started, False otherwise."
+            return old_value == 0 and value == 1
+
+        status = SubscriptionStatus(
+            self.rotation_motor.motor_done_move, check_value_sink
+        )
+        if not self.rotation_motor.motor_done_move.get():
+            raise RuntimeError("Dewar rotation motor already moving.")
+            ### Maybe don't raise an error here but rather do a timeout retry?
+        if absolute:
+            self.rotation.set(rotation_angle)
+        else:
+            current_angle = self.rotation.get()
+            self.rotation.set(current_angle + rotation_angle)
+        status.wait()
+        status = SubscriptionStatus(
+            self.rotation_motor.motor_done_move, check_value_raise
+        )
+        status.wait()
+
+class RobotArm(Device):
+    speed = Cpt(EpicsSignal, '{EMBL}:RobotSpeed')
+
+    def is_full_speed(self):
+        # Checks if the robot speed is 100%
+        if self.speed.get() < 100:
+            return False
+        return True
+