feat(robot-server, app): add a new endpoint for fast-fetching all run…

… commands (#15031)

# Overview

**Robot server changes:**

Adds a new endpoint- `GET /runs/:run_id/commandsAsPreSerializedList` to
the run commands router. This endpoint returns a list of pre-serialized
commands (that are valid json objects) of a finished run.

This endpoint is a much faster alternative to the `GET
/runs/:run_id/commands` endpoint when fetching all commands of a
completed run.

Also adds notification publishing when pre-serialized commands become
available for a run.

**App changes**

closes RQA-2645 and RQA-2647

# Risk assessment

Back-end: New endpoint so impact on existing code is close to none.
App: Medium. Fixes issues in existing behavior of handling historical runs.

---------

Co-authored-by: ncdiehl11 <[email protected]>
Co-authored-by: ncdiehl11 <[email protected]>

.eslintrc.js

@@ -52,6 +52,7 @@ module.exports = {
               'useLastRunCommandKey',
               'useCurrentMaintenanceRun',
               'useDeckConfigurationQuery',
+              'useAllCommandsAsPreSerializedList',
             ],
             message:
               'The HTTP hook is deprecated. Utilize the equivalent notification wrapper (useNotifyX) instead.',

api-client/src/runs/commands/getCommandsAsPreSerializedList.ts

@@ -0,0 +1,22 @@
+import { GET, request } from '../../request'
+
+import type { ResponsePromise } from '../../request'
+import type { HostConfig } from '../../types'
+import type {
+  CommandsAsPreSerializedListData,
+  GetCommandsParams,
+} from './types'
+
+export function getCommandsAsPreSerializedList(
+  config: HostConfig,
+  runId: string,
+  params: GetCommandsParams
+): ResponsePromise<CommandsAsPreSerializedListData> {
+  return request<CommandsAsPreSerializedListData>(
+    GET,
+    `/runs/${runId}/commandsAsPreSerializedList`,
+    null,
+    config,
+    params
+  )
+}

api-client/src/runs/commands/types.ts

@@ -34,6 +34,12 @@ export interface CommandsData {
   links: CommandsLinks
 }
 
+export interface CommandsAsPreSerializedListData {
+  data: string[]
+  meta: GetCommandsParams & { totalLength: number }
+  links: CommandsLinks
+}
+
 export interface CreateCommandParams {
   waitUntilComplete?: boolean
   timeout?: number

api-client/src/runs/index.ts

@@ -7,6 +7,7 @@ export { createCommand } from './commands/createCommand'
 export { createLiveCommand } from './commands/createLiveCommand'
 export { getCommand } from './commands/getCommand'
 export { getCommands } from './commands/getCommands'
+export { getCommandsAsPreSerializedList } from './commands/getCommandsAsPreSerializedList'
 export { createRunAction } from './createRunAction'
 export * from './createLabwareOffset'
 export * from './createLabwareDefinition'

app/src/organisms/CommandText/LoadCommandText.tsx

@@ -131,7 +131,9 @@ export const LoadCommandText = ({
           return null
         }
       } else {
-        const labware = command.result?.definition.metadata.displayName
+        const labware =
+          command.result?.definition.metadata.displayName ??
+          command.params.displayName
         return command.params.location === 'offDeck'
           ? t('load_labware_info_protocol_setup_off_deck', { labware })
           : t('load_labware_info_protocol_setup_no_module', {

app/src/organisms/RunPreview/index.tsx

@@ -4,7 +4,6 @@ import { useTranslation } from 'react-i18next'
 import { ViewportList, ViewportListRef } from 'react-viewport-list'
 
 import { RUN_STATUSES_TERMINAL } from '@opentrons/api-client'
-import { useAllCommandsQuery } from '@opentrons/react-api-client'
 import {
   ALIGN_CENTER,
   BORDERS,
@@ -21,7 +20,10 @@ import {
 } from '@opentrons/components'
 
 import { useMostRecentCompletedAnalysis } from '../LabwarePositionCheck/useMostRecentCompletedAnalysis'
-import { useNotifyLastRunCommandKey } from '../../resources/runs'
+import {
+  useNotifyAllCommandsAsPreSerializedList,
+  useNotifyLastRunCommandKey,
+} from '../../resources/runs'
 import { CommandText } from '../CommandText'
 import { Divider } from '../../atoms/structure'
 import { NAV_BAR_WIDTH } from '../../App/constants'
@@ -33,6 +35,8 @@ import type { RobotType } from '@opentrons/shared-data'
 
 const COLOR_FADE_MS = 500
 const LIVE_RUN_COMMANDS_POLL_MS = 3000
+// arbitrary large number of commands
+const MAX_COMMANDS = 100000
 
 interface RunPreviewProps {
   runId: string
@@ -52,11 +56,17 @@ export const RunPreviewComponent = (
       ? (RUN_STATUSES_TERMINAL as RunStatus[]).includes(runStatus)
       : false
   // we only ever want one request done for terminal runs because this is a heavy request
-  const commandsFromQuery = useAllCommandsQuery(runId, null, {
-    staleTime: Infinity,
-    cacheTime: Infinity,
-    enabled: isRunTerminal,
-  }).data?.data
+  const commandsFromQuery = useNotifyAllCommandsAsPreSerializedList(
+    runId,
+    { cursor: 0, pageLength: MAX_COMMANDS },
+    {
+      staleTime: Infinity,
+      cacheTime: Infinity,
+      enabled: isRunTerminal,
+    }
+  ).data?.data
+  const nullCheckedCommandsFromQuery =
+    commandsFromQuery == null ? robotSideAnalysis?.commands : commandsFromQuery
   const viewPortRef = React.useRef<HTMLDivElement | null>(null)
   const currentRunCommandKey = useNotifyLastRunCommandKey(runId, {
     refetchInterval: LIVE_RUN_COMMANDS_POLL_MS,
@@ -67,7 +77,9 @@ export const RunPreviewComponent = (
   ] = React.useState<boolean>(true)
   if (robotSideAnalysis == null) return null
   const commands =
-    (isRunTerminal ? commandsFromQuery : robotSideAnalysis.commands) ?? []
+    (isRunTerminal
+      ? nullCheckedCommandsFromQuery
+      : robotSideAnalysis.commands) ?? []
   const currentRunCommandIndex = commands.findIndex(
     c => c.key === currentRunCommandKey
   )

app/src/redux/shell/types.ts

@@ -142,6 +142,7 @@ export type NotifyTopic =
   | 'robot-server/runs'
   | `robot-server/runs/${string}`
   | 'robot-server/deck_configuration'
+  | `robot-server/runs/pre_serialized_commands/${string}`
 
 export interface NotifySubscribeAction {
   type: 'shell:NOTIFY_SUBSCRIBE'

app/src/resources/runs/index.ts

@@ -3,3 +3,4 @@ export * from './utils'
 export * from './useNotifyAllRunsQuery'
 export * from './useNotifyRunQuery'
 export * from './useNotifyLastRunCommandKey'
+export * from './useNotifyAllCommandsAsPreSerializedList'

app/src/resources/runs/useNotifyAllCommandsAsPreSerializedList.ts

@@ -0,0 +1,35 @@
+import * as React from 'react'
+
+import { useAllCommandsAsPreSerializedList } from '@opentrons/react-api-client'
+
+import { useNotifyService } from '../useNotifyService'
+
+import type { UseQueryResult } from 'react-query'
+import type { AxiosError } from 'axios'
+import type { CommandsData, GetCommandsParams } from '@opentrons/api-client'
+import type {
+  QueryOptionsWithPolling,
+  HTTPRefetchFrequency,
+} from '../useNotifyService'
+
+export function useNotifyAllCommandsAsPreSerializedList(
+  runId: string | null,
+  params?: GetCommandsParams | null,
+  options: QueryOptionsWithPolling<CommandsData, AxiosError> = {}
+): UseQueryResult<CommandsData, AxiosError> {
+  const [refetch, setRefetch] = React.useState<HTTPRefetchFrequency>(null)
+
+  useNotifyService<CommandsData, AxiosError>({
+    topic: `robot-server/runs/pre_serialized_commands/${runId}`,
+    setRefetch,
+    options,
+  })
+
+  const httpResponse = useAllCommandsAsPreSerializedList(runId, params, {
+    ...options,
+    enabled: options?.enabled !== false && refetch != null,
+    onSettled: refetch === 'once' ? () => setRefetch(null) : () => null,
+  })
+
+  return httpResponse
+}

react-api-client/src/runs/index.ts

@@ -10,6 +10,7 @@ export { usePauseRunMutation } from './usePauseRunMutation'
 export { useStopRunMutation } from './useStopRunMutation'
 export { useRunActionMutations } from './useRunActionMutations'
 export { useAllCommandsQuery } from './useAllCommandsQuery'
+export { useAllCommandsAsPreSerializedList } from './useAllCommandsAsPreSerializedList'
 export { useCommandQuery } from './useCommandQuery'
 export * from './useCreateLabwareOffsetMutation'
 export * from './useCreateLabwareDefinitionMutation'

react-api-client/src/runs/useAllCommandsAsPreSerializedList.ts

@@ -0,0 +1,52 @@
+import { UseQueryResult, useQuery } from 'react-query'
+import { getCommandsAsPreSerializedList } from '@opentrons/api-client'
+import { useHost } from '../api'
+import type { UseQueryOptions } from 'react-query'
+import type {
+  GetCommandsParams,
+  HostConfig,
+  CommandsData,
+  RunCommandSummary,
+} from '@opentrons/api-client'
+
+const DEFAULT_PAGE_LENGTH = 30
+export const DEFAULT_PARAMS: GetCommandsParams = {
+  cursor: null,
+  pageLength: DEFAULT_PAGE_LENGTH,
+}
+
+export function useAllCommandsAsPreSerializedList<TError = Error>(
+  runId: string | null,
+  params?: GetCommandsParams | null,
+  options: UseQueryOptions<CommandsData, TError> = {}
+): UseQueryResult<CommandsData, TError> {
+  const host = useHost()
+  const nullCheckedParams = params ?? DEFAULT_PARAMS
+
+  const allOptions: UseQueryOptions<CommandsData, TError> = {
+    ...options,
+    enabled: host !== null && runId != null && options.enabled !== false,
+  }
+  const { cursor, pageLength } = nullCheckedParams
+  const query = useQuery<CommandsData, TError>(
+    [host, 'runs', runId, 'getCommandsAsPreSerializedList', cursor, pageLength],
+    () => {
+      return getCommandsAsPreSerializedList(
+        host as HostConfig,
+        runId as string,
+        nullCheckedParams
+      ).then(response => {
+        const responseData = response.data
+        return {
+          ...responseData,
+          data: responseData.data.map(
+            command => JSON.parse(command) as RunCommandSummary
+          ),
+        }
+      })
+    },
+    allOptions
+  )
+
+  return query
+}

robot-server/robot_server/runs/router/actions_router.py

@@ -28,6 +28,7 @@
     MaintenanceEngineStore,
 )
 from robot_server.maintenance_runs.dependencies import get_maintenance_engine_store
+from robot_server.service.notifications import get_runs_publisher, RunsPublisher
 
 log = logging.getLogger(__name__)
 actions_router = APIRouter()
@@ -45,6 +46,7 @@ async def get_run_controller(
     task_runner: TaskRunner = Depends(get_task_runner),
     engine_store: EngineStore = Depends(get_engine_store),
     run_store: RunStore = Depends(get_run_store),
+    runs_publisher: RunsPublisher = Depends(get_runs_publisher),
 ) -> RunController:
     """Get a RunController for the current run.
 
@@ -67,6 +69,7 @@ async def get_run_controller(
         task_runner=task_runner,
         engine_store=engine_store,
         run_store=run_store,
+        runs_publisher=runs_publisher,
     )
 
 

robot-server/robot_server/runs/router/commands_router.py

@@ -6,6 +6,7 @@
 
 from anyio import move_on_after
 from fastapi import APIRouter, Depends, Query, status
+
 from pydantic import BaseModel, Field
 
 from opentrons.protocol_engine import (
@@ -21,11 +22,12 @@
     MultiBody,
     MultiBodyMeta,
     PydanticResponse,
+    SimpleMultiBody,
 )
 from robot_server.robot.control.dependencies import require_estop_in_good_state
 
 from ..run_models import RunCommandSummary
-from ..run_data_manager import RunDataManager
+from ..run_data_manager import RunDataManager, PreSerializedCommandsNotAvailableError
 from ..engine_store import EngineStore
 from ..run_store import RunStore, CommandNotFoundError
 from ..run_models import RunNotFoundError
@@ -70,6 +72,18 @@ class CommandNotAllowed(ErrorDetails):
     title: str = "Command Not Allowed"
 
 
+class PreSerializedCommandsNotAvailable(ErrorDetails):
+    """An error if one tries to fetch pre-serialized commands before they are written to the database."""
+
+    id: Literal[
+        "PreSerializedCommandsNotAvailable"
+    ] = "PreSerializedCommandsNotAvailable"
+    title: str = "Pre-Serialized commands not available."
+    detail: str = (
+        "Pre-serialized commands are only available once a run has finished running."
+    )
+
+
 class CommandLinkMeta(BaseModel):
     """Metadata about a command resource referenced in `links`."""
 
@@ -351,6 +365,56 @@ async def get_run_commands(
     )
 
 
+# TODO (spp, 2024-05-01): explore alternatives to returning commands as list of strings.
+#                Options: 1. JSON Lines
+#                         2. Simple de-serialized commands list w/o pydantic model conversion
+@PydanticResponse.wrap_route(
+    commands_router.get,
+    path="/runs/{runId}/commandsAsPreSerializedList",
+    summary="Get all commands of a completed run as a list of pre-serialized commands",
+    description=(
+        "Get all commands of a completed run as a list of pre-serialized commands."
+        "**Warning:** This endpoint is experimental. We may change or remove it without warning."
+        "\n\n"
+        "The commands list will only be available after a run has completed"
+        " (whether successful, failed or stopped) and its data has been committed to the database."
+        " If a request is received before the run is completed, it will return a 503 Unavailable error."
+        " This is a faster alternative to fetching the full commands list using"
+        " `GET /runs/{runId}/commands`. For large protocols (10k+ commands), the above"
+        " endpoint can take minutes to respond, whereas this one should only take a few seconds."
+    ),
+    responses={
+        status.HTTP_404_NOT_FOUND: {"model": ErrorBody[RunNotFound]},
+        status.HTTP_503_SERVICE_UNAVAILABLE: {
+            "model": ErrorBody[PreSerializedCommandsNotAvailable]
+        },
+    },
+)
+async def get_run_commands_as_pre_serialized_list(
+    runId: str,
+    run_data_manager: RunDataManager = Depends(get_run_data_manager),
+) -> PydanticResponse[SimpleMultiBody[str]]:
+    """Get all commands of a completed run as a list of pre-serialized (string encoded) commands.
+
+    Arguments:
+        runId: Requested run ID, from the URL
+        run_data_manager: Run data retrieval interface.
+    """
+    try:
+        commands = run_data_manager.get_all_commands_as_preserialized_list(runId)
+    except RunNotFoundError as e:
+        raise RunNotFound.from_exc(e).as_error(status.HTTP_404_NOT_FOUND) from e
+    except PreSerializedCommandsNotAvailableError as e:
+        raise PreSerializedCommandsNotAvailable.from_exc(e).as_error(
+            status.HTTP_503_SERVICE_UNAVAILABLE
+        ) from e
+    return await PydanticResponse.create(
+        content=SimpleMultiBody.construct(
+            data=commands, meta=MultiBodyMeta(cursor=0, totalLength=len(commands))
+        )
+    )
+
+
 @PydanticResponse.wrap_route(
     commands_router.get,
     path="/runs/{runId}/commands/{commandId}",

robot-server/robot_server/runs/run_controller.py

@@ -13,6 +13,8 @@
 
 from opentrons.protocol_engine.types import DeckConfigurationType
 
+from robot_server.service.notifications import RunsPublisher
+
 log = logging.getLogger(__name__)
 
 
@@ -21,19 +23,21 @@ class RunActionNotAllowedError(RoboticsInteractionError):
 
 
 class RunController:
-    """An interface to manage the side-effects of requested run actions."""
+    """An interface to manage the side effects of requested run actions."""
 
     def __init__(
         self,
         run_id: str,
         task_runner: TaskRunner,
         engine_store: EngineStore,
         run_store: RunStore,
+        runs_publisher: RunsPublisher,
     ) -> None:
         self._run_id = run_id
         self._task_runner = task_runner
         self._engine_store = engine_store
         self._run_store = run_store
+        self._runs_publisher = runs_publisher
 
     def create_action(
         self,
@@ -108,3 +112,6 @@ async def _run_protocol_and_insert_result(
             commands=result.commands,
             run_time_parameters=result.parameters,
         )
+        await self._runs_publisher.publish_pre_serialized_commands_notification(
+            self._run_id
+        )