feat(hardware): implement multipass lld (#15122)

<!--
Thanks for taking the time to open a pull request! Please make sure
you've read the "Opening Pull Requests" section of our Contributing
Guide:


https://github.com/Opentrons/opentrons/blob/edge/CONTRIBUTING.md#opening-pull-requests

To ensure your code is reviewed quickly and thoroughly, please fill out
the sections below to the best of your ability!
-->

# Overview
Due to limits on how far the z stage can move depending on a combination
of factors including z speed, plunger speed and labware depth we may
need to run the probe multiple times in order to find the liquid.

This PR lets the liquid probe script handle when the liquid is not found
and start a new pass that starts at the z height the the previous pass
stopped at.


<!--
Use this section to describe your pull-request at a high level. If the
PR addresses any open issues, please tag the issues here.
-->

# Test Plan

<!--
Use this section to describe the steps that you took to test your Pull
Request.
If you did not perform any testing provide justification why.

OT-3 Developers: You should default to testing on actual physical
hardware.
Once again, if you did not perform testing against hardware, justify
why.

Note: It can be helpful to write a test plan before doing development

Example Test Plan (HTTP API Change)

- Verified that new optional argument `dance-party` causes the robot to
flash its lights, move the pipettes,
then home.
- Verified that when you omit the `dance-party` option the robot homes
normally
- Added protocol that uses `dance-party` argument to G-Code Testing
Suite
- Ran protocol that did not use `dance-party` argument and everything
was successful
- Added unit tests to validate that changes to pydantic model are
correct

-->

# Changelog

<!--
List out the changes to the code in this PR. Please try your best to
categorize your changes and describe what has changed and why.

Example changelog:
- Fixed app crash when trying to calibrate an illegal pipette
- Added state to API to track pipette usage
- Updated API docs to mention only two pipettes are supported

IMPORTANT: MAKE SURE ANY BREAKING CHANGES ARE PROPERLY COMMUNICATED
-->

# Review requests

<!--
Describe any requests for your reviewers here.
-->

# Risk assessment

<!--
Carefully go over your pull request and look at the other parts of the
codebase it may affect. Look for the possibility, even if you think it's
small, that your change may affect some other part of the system - for
instance, changing return tip behavior in protocol may also change the
behavior of labware calibration.

Identify the other parts of the system your codebase may affect, so that
in addition to your own review and testing, other people who may not
have the system internalized as much as you can focus their attention
and testing there.
-->

api/src/opentrons/hardware_control/backends/ot3controller.py

@@ -191,6 +191,7 @@
     PipetteOverpressureError,
     FirmwareUpdateRequiredError,
     FailedGripperPickupError,
+    LiquidNotFoundError,
 )
 
 from .subsystem_manager import SubsystemManager
@@ -1399,6 +1400,18 @@ async def liquid_probe(
         for node, point in positions.items():
             self._position.update({node: point.motor_position})
             self._encoder_position.update({node: point.encoder_position})
+        if (
+            head_node not in positions
+            or positions[head_node].move_ack
+            == MoveCompleteAck.complete_without_condition
+        ):
+            raise LiquidNotFoundError(
+                "Liquid not found during probe.",
+                {
+                    str(node_to_axis(node)): str(point.motor_position)
+                    for node, point in positions.items()
+                },
+            )
         return self._position[axis_to_node(Axis.by_mount(mount))]
 
     async def capacitive_probe(

api/src/opentrons/hardware_control/ot3api.py

@@ -2556,9 +2556,7 @@ async def liquid_probe(
         reading from the pressure sensor.
 
         If the move is completed without the specified threshold being triggered, a
-        LiquidNotFound error will be thrown.
-        If the threshold is triggered before the minimum z distance has been traveled,
-        a EarlyLiquidSenseTrigger error will be thrown.
+        LiquidNotFoundError error will be thrown.
 
         Otherwise, the function will stop moving once the threshold is triggered,
         and return the position of the

api/src/opentrons/hardware_control/types.py

@@ -694,19 +694,6 @@ def __init__(
         )
 
 
-class LiquidNotFound(RuntimeError):
-    """Error raised if liquid sensing move completes without detecting liquid."""
-
-    def __init__(
-        self, position: Dict[Axis, float], max_z_pos: Dict[Axis, float]
-    ) -> None:
-        """Initialize LiquidNotFound error."""
-        super().__init__(
-            f"Liquid threshold not found, current_position = {position}"
-            f"position at max travel allowed = {max_z_pos}"
-        )
-
-
 class FailedTipStateCheck(RuntimeError):
     """Error raised if the tip ejector state does not match the expected value."""
 

api/tests/opentrons/hardware_control/backends/test_ot3_controller.py

@@ -95,6 +95,7 @@
     EStopNotPresentError,
     FirmwareUpdateRequiredError,
     FailedGripperPickupError,
+    LiquidNotFoundError,
 )
 
 from opentrons_hardware.hardware_control.move_group_runner import MoveGroupRunner
@@ -715,14 +716,19 @@ async def test_liquid_probe(
     mock_move_group_run: mock.AsyncMock,
     mock_send_stop_threshold: mock.AsyncMock,
 ) -> None:
-    await controller.liquid_probe(
-        mount=mount,
-        max_z_distance=fake_liquid_settings.max_z_distance,
-        mount_speed=fake_liquid_settings.mount_speed,
-        plunger_speed=fake_liquid_settings.plunger_speed,
-        threshold_pascals=fake_liquid_settings.sensor_threshold_pascals,
-        output_option=fake_liquid_settings.output_option,
-    )
+    try:
+        await controller.liquid_probe(
+            mount=mount,
+            max_z_distance=fake_liquid_settings.max_z_distance,
+            mount_speed=fake_liquid_settings.mount_speed,
+            plunger_speed=fake_liquid_settings.plunger_speed,
+            threshold_pascals=fake_liquid_settings.sensor_threshold_pascals,
+            output_option=fake_liquid_settings.output_option,
+        )
+    except LiquidNotFoundError:
+        # the move raises a liquid not found now since we don't call the move group and it doesn't
+        # get any positions back
+        pass
     move_groups = (mock_move_group_run.call_args_list[0][0][0]._move_groups)[0][0]
     head_node = axis_to_node(Axis.by_mount(mount))
     tool_node = sensor_node_for_mount(mount)

hardware-testing/hardware_testing/liquid_sense/__main__.py

@@ -94,8 +94,9 @@ class RunArgs:
     start_height_offset: float
     aspirate: bool
     dial_indicator: Optional[mitutoyo_digimatic_indicator.Mitutoyo_Digimatic_Indicator]
-    plunger_speed: bool
+    plunger_speed: float
     trials_before_jog: int
+    multi_passes: int
 
     @classmethod
     def _get_protocol_context(cls, args: argparse.Namespace) -> ProtocolContext:
@@ -236,6 +237,7 @@ def build_run_args(cls, args: argparse.Namespace) -> "RunArgs":
             dial_indicator=dial,
             plunger_speed=args.plunger_speed,
             trials_before_jog=args.trials_before_jog,
+            multi_passes=args.multi_passes,
         )
 
 
@@ -266,6 +268,7 @@ def build_run_args(cls, args: argparse.Namespace) -> "RunArgs":
     parser.add_argument("--ignore-env", action="store_true")
     parser.add_argument("--ignore-dial", action="store_true")
     parser.add_argument("--trials-before-jog", type=int, default=10)
+    parser.add_argument("--multi-passes", type=int, default=1)
 
     args = parser.parse_args()
     run_args = RunArgs.build_run_args(args)

hardware-testing/hardware_testing/liquid_sense/execute.py

@@ -26,6 +26,15 @@
 
 from opentrons.protocol_api import ProtocolContext, Well, Labware
 
+from opentrons_shared_data.errors.exceptions import LiquidNotFoundError
+
+
+PROBE_MAX_TIME: Dict[int, float] = {
+    1: 2.75,
+    8: 1.75,
+    96: 0.85,
+}
+
 
 def _load_tipracks(
     ctx: ProtocolContext, pipette_channels: int, protocol_cfg: Any, tip: int
@@ -280,6 +289,41 @@ def _get_target_height() -> float:
     store_tip_results(run_args.test_report, tip, results, adjusted_results)
 
 
+def get_plunger_travel(run_args: RunArgs) -> float:
+    """Get the travel distance for the pipette."""
+    hw_mount = OT3Mount.LEFT if run_args.pipette.mount == "left" else OT3Mount.RIGHT
+    hw_api = get_sync_hw_api(run_args.ctx)
+    plunger_positions = hw_api._pipette_handler.get_pipette(hw_mount).plunger_positions
+    plunger_travel = plunger_positions.bottom - plunger_positions.top
+    return plunger_travel
+
+
+def find_max_z_distances(
+    run_args: RunArgs, tip: int, well: Well, p_speed: float
+) -> List[float]:
+    """Returns a list of max z distances for each probe.
+
+    Each element is the max travel for the z mount for a particular call
+    to hw_api.liquid_probe, it is the limit of z distance the pipette can
+    move with the combination of z speed and plunger speed,
+    if the distance would exceed the well depth then the number is
+    truncated to avoid collisions.
+    """
+    z_speed = run_args.z_speed
+    max_z_distance = well.depth + run_args.start_height_offset
+    plunger_travel = get_plunger_travel(run_args)
+    p_travel_time = min(
+        plunger_travel / p_speed, PROBE_MAX_TIME[run_args.pipette_channels]
+    )
+
+    z_travels: List[float] = []
+    while max_z_distance > 0:
+        next_travel = min(p_travel_time * z_speed, max_z_distance)
+        z_travels.append(next_travel)
+        max_z_distance -= next_travel
+    return z_travels
+
+
 def _run_trial(run_args: RunArgs, tip: int, well: Well, trial: int) -> float:
     hw_api = get_sync_hw_api(run_args.ctx)
     lqid_cfg: Dict[str, int] = LIQUID_PROBE_SETTINGS[run_args.pipette_volume][
@@ -303,25 +347,37 @@ def _run_trial(run_args: RunArgs, tip: int, well: Well, trial: int) -> float:
         if run_args.plunger_speed == -1
         else run_args.plunger_speed
     )
-    lps = LiquidProbeSettings(
-        starting_mount_height=well.top().point.z + run_args.start_height_offset,
-        max_z_distance=min(well.depth, lqid_cfg["max_z_distance"]),
-        min_z_distance=lqid_cfg["min_z_distance"],
-        mount_speed=run_args.z_speed,
-        plunger_speed=plunger_speed,
-        sensor_threshold_pascals=lqid_cfg["sensor_threshold_pascals"],
-        expected_liquid_height=110,
-        output_option=OutputOptions.sync_buffer_to_csv,
-        aspirate_while_sensing=run_args.aspirate,
-        auto_zero_sensor=True,
-        num_baseline_reads=10,
-        data_files=data_files,
-    )
 
-    hw_mount = OT3Mount.LEFT if run_args.pipette.mount == "left" else OT3Mount.RIGHT
-    run_args.recorder.set_sample_tag(f"trial-{trial}-{tip}ul")
-    # TODO add in stuff for secondary probe
-    height = hw_api.liquid_probe(hw_mount, lps, probe_target)
+    z_distances: List[float] = find_max_z_distances(run_args, tip, well, plunger_speed)
+    z_distances = z_distances[: run_args.multi_passes]
+    start_height = well.top().point.z + run_args.start_height_offset
+    for z_dist in z_distances:
+        lps = LiquidProbeSettings(
+            starting_mount_height=start_height,
+            max_z_distance=z_dist,
+            min_z_distance=lqid_cfg["min_z_distance"],
+            mount_speed=run_args.z_speed,
+            plunger_speed=plunger_speed,
+            sensor_threshold_pascals=lqid_cfg["sensor_threshold_pascals"],
+            expected_liquid_height=110,
+            output_option=OutputOptions.sync_buffer_to_csv,
+            aspirate_while_sensing=run_args.aspirate,
+            auto_zero_sensor=True,
+            num_baseline_reads=10,
+            data_files=data_files,
+        )
+
+        hw_mount = OT3Mount.LEFT if run_args.pipette.mount == "left" else OT3Mount.RIGHT
+        run_args.recorder.set_sample_tag(f"trial-{trial}-{tip}ul")
+        # TODO add in stuff for secondary probe
+        try:
+            height = hw_api.liquid_probe(hw_mount, lps, probe_target)
+        except LiquidNotFoundError as lnf:
+            ui.print_info(f"Liquid not found current position {lnf.detail}")
+            start_height -= z_dist
+        else:
+            break
+        run_args.recorder.clear_sample_tag()
+
     ui.print_info(f"Trial {trial} complete")
-    run_args.recorder.clear_sample_tag()
     return height

hardware/opentrons_hardware/hardware_control/tool_sensors.py

@@ -154,7 +154,7 @@ async def run_sync_buffer_to_csv(
                     )
                 ),
             )
-            await asyncio.sleep(10)
+            await sensor_capturer.wait_for_complete()
             messenger.remove_listener(sensor_capturer)
         await messenger.send(
             node_id=tool,

hardware/opentrons_hardware/sensors/sensor_driver.py

@@ -235,6 +235,7 @@ def __init__(
         self.response_queue: asyncio.Queue[float] = asyncio.Queue()
         self.mount = mount
         self.start_time = 0.0
+        self.event: Any = None
 
     async def __aenter__(self) -> None:
         """Create a csv heading for logging pressure readings."""
@@ -248,6 +249,16 @@ async def __aexit__(self, *args: Any) -> None:
         """Close csv file."""
         self.data_file.close()
 
+    async def wait_for_complete(self, wait_time: float = 2.0) -> None:
+        """Wait for the data to stop, only use this with a send_accumulated_data_request."""
+        self.event = asyncio.Event()
+        recieving = True
+        while recieving:
+            await asyncio.sleep(wait_time)
+            recieving = self.event.is_set()
+            self.event.clear()
+        self.event = None
+
     def __call__(
         self,
         message: MessageDefinition,
@@ -261,3 +272,5 @@ def __call__(
             self.response_queue.put_nowait(data)
             current_time = round((time.time() - self.start_time), 3)
             self.csv_writer.writerow([current_time, data])  # type: ignore
+            if self.event is not None:
+                self.event.set()

shared-data/errors/definitions/1/errors.json

@@ -122,6 +122,10 @@
       "detail": "Motor Driver Error",
       "category": "roboticsControlError"
     },
+    "2017": {
+      "detail": "Liquid Not Found",
+      "category": "roboticsControlError"
+    },
     "3000": {
       "detail": "A robotics interaction error occurred.",
       "category": "roboticsInteractionError"

shared-data/python/opentrons_shared_data/errors/codes.py

@@ -60,6 +60,7 @@ class ErrorCodes(Enum):
     EXECUTION_CANCELLED = _code_from_dict_entry("2014")
     FAILED_GRIPPER_PICKUP_ERROR = _code_from_dict_entry("2015")
     MOTOR_DRIVER_ERROR = _code_from_dict_entry("2016")
+    LIQUID_NOT_FOUND = _code_from_dict_entry("2017")
     ROBOTICS_INTERACTION_ERROR = _code_from_dict_entry("3000")
     LABWARE_DROPPED = _code_from_dict_entry("3001")
     LABWARE_NOT_PICKED_UP = _code_from_dict_entry("3002")

shared-data/python/opentrons_shared_data/errors/exceptions.py

@@ -611,6 +611,24 @@ def __init__(
         super().__init__(ErrorCodes.MOTOR_DRIVER_ERROR, message, detail, wrapping)
 
 
+class LiquidNotFoundError(RoboticsControlError):
+    """Error raised if liquid sensing move completes without detecting liquid."""
+
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        detail: Optional[Dict[str, str]] = None,
+        wrapping: Optional[Sequence[EnumeratedError]] = None,
+    ) -> None:
+        """Initialize LiquidNotFoundError."""
+        super().__init__(
+            ErrorCodes.LIQUID_NOT_FOUND,
+            message,
+            detail,
+            wrapping,
+        )
+
+
 class LabwareDroppedError(RoboticsInteractionError):
     """An error indicating that the gripper dropped labware it was holding."""