fix(api-client, react-api-client, app, robot-server): support multiple recovery policies during a run

api-client/src/runs/getErrorRecoveryPolicy.ts

@@ -0,0 +1,17 @@
+import { GET, request } from '../request'
+
+import type { HostConfig } from '../types'
+import type { ResponsePromise } from '../request'
+import type { ErrorRecoveryPolicyResponse } from './types'
+
+export function getErrorRecoveryPolicy(
+  config: HostConfig,
+  runId: string
+): ResponsePromise<ErrorRecoveryPolicyResponse> {
+  return request<ErrorRecoveryPolicyResponse>(
+    GET,
+    `/get/${runId}/errorRecoveryPolicy`,
+    null,
+    config
+  )
+}

api-client/src/runs/index.ts

@@ -15,6 +15,7 @@ export * from './createLabwareOffset'
 export * from './createLabwareDefinition'
 export * from './constants'
 export * from './updateErrorRecoveryPolicy'
+export * from './getErrorRecoveryPolicy'
 
 export * from './types'
 export type { CreateRunData } from './createRun'

api-client/src/runs/types.ts

@@ -204,6 +204,7 @@ export interface UpdateErrorRecoveryPolicyRequest {
 }
 
 export type UpdateErrorRecoveryPolicyResponse = Record<string, never>
+export type ErrorRecoveryPolicyResponse = ErrorRecoveryPolicy
 
 /**
  * Current Run State Data

api-client/src/runs/updateErrorRecoveryPolicy.ts

@@ -6,7 +6,7 @@ import type {
   ErrorRecoveryPolicy,
   IfMatchType,
   UpdateErrorRecoveryPolicyRequest,
-  UpdateErrorRecoveryPolicyResponse,
+  ErrorRecoveryPolicyResponse,
 } from './types'
 import type { RunCommandError, RunTimeCommand } from '@opentrons/shared-data'
 
@@ -20,13 +20,15 @@ export function updateErrorRecoveryPolicy(
   config: HostConfig,
   runId: string,
   policyRules: RecoveryPolicyRulesParams
-): ResponsePromise<UpdateErrorRecoveryPolicyResponse> {
+): ResponsePromise<ErrorRecoveryPolicyResponse> {
   const policy = buildErrorRecoveryPolicyBody(policyRules)
 
-  return request<
-    UpdateErrorRecoveryPolicyResponse,
-    UpdateErrorRecoveryPolicyRequest
-  >(PUT, `/runs/${runId}/errorRecoveryPolicy`, { data: policy }, config)
+  return request<ErrorRecoveryPolicyResponse, UpdateErrorRecoveryPolicyRequest>(
+    PUT,
+    `/runs/${runId}/errorRecoveryPolicy`,
+    { data: policy },
+    config
+  )
 }
 
 function buildErrorRecoveryPolicyBody(

app/src/organisms/ErrorRecoveryFlows/hooks/__tests__/useRecoveryCommands.test.ts

@@ -4,11 +4,13 @@ import { renderHook, act } from '@testing-library/react'
 import {
   useResumeRunFromRecoveryMutation,
   useStopRunMutation,
-  useUpdateErrorRecoveryPolicy,
   useResumeRunFromRecoveryAssumingFalsePositiveMutation,
 } from '@opentrons/react-api-client'
 
-import { useChainRunCommands } from '/app/resources/runs'
+import {
+  useChainRunCommands,
+  useUpdateRecoveryPolicyWithStrategy,
+} from '/app/resources/runs'
 import {
   useRecoveryCommands,
   HOME_PIPETTE_Z_AXES,
@@ -80,9 +82,9 @@ describe('useRecoveryCommands', () => {
     vi.mocked(useChainRunCommands).mockReturnValue({
       chainRunCommands: mockChainRunCommands,
     } as any)
-    vi.mocked(useUpdateErrorRecoveryPolicy).mockReturnValue({
-      mutateAsync: mockUpdateErrorRecoveryPolicy,
-    } as any)
+    vi.mocked(useUpdateRecoveryPolicyWithStrategy).mockReturnValue(
+      mockUpdateErrorRecoveryPolicy as any
+    )
     vi.mocked(
       useResumeRunFromRecoveryAssumingFalsePositiveMutation
     ).mockReturnValue({
@@ -361,7 +363,8 @@ describe('useRecoveryCommands', () => {
     )
 
     expect(mockUpdateErrorRecoveryPolicy).toHaveBeenCalledWith(
-      expectedPolicyRules
+      expectedPolicyRules,
+      'append'
     )
   })
 

app/src/organisms/ErrorRecoveryFlows/hooks/useRecoveryCommands.ts

@@ -4,11 +4,13 @@ import head from 'lodash/head'
 import {
   useResumeRunFromRecoveryMutation,
   useStopRunMutation,
-  useUpdateErrorRecoveryPolicy,
   useResumeRunFromRecoveryAssumingFalsePositiveMutation,
 } from '@opentrons/react-api-client'
 
-import { useChainRunCommands } from '/app/resources/runs'
+import {
+  useChainRunCommands,
+  useUpdateRecoveryPolicyWithStrategy,
+} from '/app/resources/runs'
 import { DEFINED_ERROR_TYPES, ERROR_KINDS, RECOVERY_MAP } from '../constants'
 import { getErrorKind } from '/app/organisms/ErrorRecoveryFlows/utils'
 
@@ -23,12 +25,7 @@ import type {
   PrepareToAspirateRunTimeCommand,
   MoveLabwareParams,
 } from '@opentrons/shared-data'
-import type {
-  CommandData,
-  IfMatchType,
-  RecoveryPolicyRulesParams,
-  RunAction,
-} from '@opentrons/api-client'
+import type { CommandData, IfMatchType, RunAction } from '@opentrons/api-client'
 import type { WellGroup } from '@opentrons/components'
 import type { FailedCommand, RecoveryRoute, RouteStep } from '../types'
 import type { UseFailedLabwareUtilsResult } from './useFailedLabwareUtils'
@@ -38,6 +35,7 @@ import type { UseRecoveryAnalyticsResult } from '/app/redux-resources/analytics'
 import type { CurrentRecoveryOptionUtils } from './useRecoveryRouting'
 import type { ErrorRecoveryFlowsProps } from '..'
 import type { FailedCommandBySource } from './useRetainedFailedCommandBySource'
+import type { UpdateErrorRecoveryPolicyWithStrategy } from '/app/resources/runs'
 
 interface UseRecoveryCommandsParams {
   runId: string
@@ -100,9 +98,7 @@ export function useRecoveryCommands({
     mutateAsync: resumeRunFromRecoveryAssumingFalsePositive,
   } = useResumeRunFromRecoveryAssumingFalsePositiveMutation()
   const { stopRun } = useStopRunMutation()
-  const {
-    mutateAsync: updateErrorRecoveryPolicy,
-  } = useUpdateErrorRecoveryPolicy(runId)
+  const updateErrorRecoveryPolicy = useUpdateRecoveryPolicyWithStrategy(runId)
   const { makeSuccessToast } = recoveryToastUtils
 
   // TODO(jh, 11-21-24): Some commands return a 200 with an error body. We should catch these and propagate the error.
@@ -231,7 +227,7 @@ export function useRecoveryCommands({
           ifMatch
         )
 
-        return updateErrorRecoveryPolicy(ignorePolicyRules)
+        return updateErrorRecoveryPolicy(ignorePolicyRules, 'append')
           .then(() => Promise.resolve())
           .catch(() =>
             Promise.reject(new Error('Failed to update recovery policy.'))
@@ -421,12 +417,8 @@ export const buildIgnorePolicyRules = (
   commandType: FailedCommand['commandType'],
   errorType: string,
   ifMatch: IfMatchType
-): RecoveryPolicyRulesParams => {
-  return [
-    {
-      commandType,
-      errorType,
-      ifMatch,
-    },
-  ]
-}
+): UpdateErrorRecoveryPolicyWithStrategy['newPolicy'] => ({
+  commandType,
+  errorType,
+  ifMatch,
+})

app/src/resources/runs/index.ts

@@ -29,3 +29,4 @@ export * from './useModuleCalibrationStatus'
 export * from './useProtocolAnalysisErrors'
 export * from './useLastRunCommand'
 export * from './useRunStatuses'
+export * from './useUpdateRecoveryPolicyWithStrategy'

app/src/resources/runs/useUpdateRecoveryPolicyWithStrategy.ts

@@ -0,0 +1,60 @@
+import {
+  useHost,
+  useUpdateErrorRecoveryPolicy,
+} from '@opentrons/react-api-client'
+import { getErrorRecoveryPolicy } from '@opentrons/api-client'
+
+import type {
+  HostConfig,
+  RecoveryPolicyRulesParams,
+  UpdateErrorRecoveryPolicyResponse,
+} from '@opentrons/api-client'
+
+/**
+ * append - Add a new policy rule to the end of the existing recovery policy.
+ */
+export type UpdatePolicyStrategy = 'append'
+
+export interface UpdateErrorRecoveryPolicyWithStrategy {
+  runId: string
+  newPolicy: RecoveryPolicyRulesParams[number]
+  strategy: UpdatePolicyStrategy
+}
+
+export function useUpdateRecoveryPolicyWithStrategy(
+  runId: string
+): (
+  newPolicy: UpdateErrorRecoveryPolicyWithStrategy['newPolicy'],
+  strategy: UpdateErrorRecoveryPolicyWithStrategy['strategy']
+) => Promise<UpdateErrorRecoveryPolicyResponse> {
+  const host = useHost()
+
+  const {
+    mutateAsync: updateErrorRecoveryPolicy,
+  } = useUpdateErrorRecoveryPolicy(runId)
+
+  return (
+    newPolicy: UpdateErrorRecoveryPolicyWithStrategy['newPolicy'],
+    strategy: UpdateErrorRecoveryPolicyWithStrategy['strategy']
+  ) =>
+    getErrorRecoveryPolicy(host as HostConfig, runId).then(res => {
+      const existingPolicyRules = res.data.policyRules.map(rule => ({
+        commandType: rule.matchCriteria.command.commandType,
+        errorType: rule.matchCriteria.command.error.errorType,
+        ifMatch: rule.ifMatch,
+      }))
+
+      const buildUpdatedPolicy = (): RecoveryPolicyRulesParams => {
+        switch (strategy) {
+          case 'append':
+            return [...existingPolicyRules, newPolicy]
+          default: {
+            console.error('Handled policy strategy, using append.')
+            return [...existingPolicyRules, newPolicy]
+          }
+        }
+      }
+
+      return updateErrorRecoveryPolicy(buildUpdatedPolicy())
+    })
+}

react-api-client/src/runs/index.ts

@@ -19,6 +19,7 @@ export { useRunCommandErrors } from './useRunCommandErrors'
 export * from './useCreateLabwareOffsetMutation'
 export * from './useCreateLabwareDefinitionMutation'
 export * from './useUpdateErrorRecoveryPolicy'
+export * from './useErrorRecoveryPolicy'
 
 export type { UsePlayRunMutationResult } from './usePlayRunMutation'
 export type { UsePauseRunMutationResult } from './usePauseRunMutation'

react-api-client/src/runs/useErrorRecoveryPolicy.ts

@@ -0,0 +1,31 @@
+import { useQuery } from 'react-query'
+
+import { getErrorRecoveryPolicy } from '@opentrons/api-client'
+
+import { useHost } from '../api'
+
+import type { UseQueryOptions, UseQueryResult } from 'react-query'
+import type {
+  ErrorRecoveryPolicyResponse,
+  HostConfig,
+} from '@opentrons/api-client'
+
+export function useErrorRecoveryPolicy(
+  runId: string,
+  options: UseQueryOptions<ErrorRecoveryPolicyResponse, Error> = {}
+): UseQueryResult<ErrorRecoveryPolicyResponse, Error> {
+  const host = useHost()
+
+  const query = useQuery<ErrorRecoveryPolicyResponse, Error>(
+    [host, 'runs', runId, 'errorRecoveryPolicy'],
+    () =>
+      getErrorRecoveryPolicy(host as HostConfig, runId)
+        .then(response => response.data)
+        .catch(e => {
+          throw e
+        }),
+    options
+  )
+
+  return query
+}

react-api-client/src/runs/useUpdateErrorRecoveryPolicy.ts

@@ -16,7 +16,7 @@
   HostConfig,
 } from '@opentrons/api-client'
 
-export type UseUpdateErrorRecoveryPolicyResponse = UseMutationResult<
+export type UseErrorRecoveryPolicyResponse = UseMutationResult<
   UpdateErrorRecoveryPolicyResponse,
   AxiosError,
   RecoveryPolicyRulesParams
@@ -37,7 +37,7 @@
 export function useUpdateErrorRecoveryPolicy(
   runId: string,
   options: UseUpdateErrorRecoveryPolicyOptions = {}
-): UseUpdateErrorRecoveryPolicyResponse {
+): UseErrorRecoveryPolicyResponse {
   const host = useHost()
 
   const mutation = useMutation<
@@ -46,7 +46,7 @@
     RecoveryPolicyRulesParams
   >(
     [host, 'runs', runId, 'errorRecoveryPolicy'],
     (policyRules: RecoveryPolicyRulesParams) =>
       updateErrorRecoveryPolicy(host as HostConfig, runId, policyRules)
         .then(response => response.data)
         .catch(e => {

robot-server/robot_server/runs/dependencies.py

@@ -48,6 +48,7 @@
 _run_orchestrator_store_accessor = AppStateAccessor[RunOrchestratorStore](
     "run_orchestrator_store"
 )
+_run_data_manager_accessor = AppStateAccessor[RunDataManager]("run_data_manager")
 _light_control_accessor = AppStateAccessor[LightController]("light_controller")
 
 
@@ -154,6 +155,7 @@ async def get_is_okay_to_create_maintenance_run(
 
 
 async def get_run_data_manager(
+    app_state: Annotated[AppState, Depends(get_app_state)],
     task_runner: Annotated[TaskRunner, Depends(get_task_runner)],
     run_orchestrator_store: Annotated[
         RunOrchestratorStore, Depends(get_run_orchestrator_store)
@@ -164,14 +166,20 @@ async def get_run_data_manager(
         ErrorRecoverySettingStore, Depends(get_error_recovery_setting_store)
     ],
 ) -> RunDataManager:
-    """Get a run data manager to keep track of current/historical run data."""
-    return RunDataManager(
-        run_orchestrator_store=run_orchestrator_store,
-        run_store=run_store,
-        error_recovery_setting_store=error_recovery_setting_store,
-        task_runner=task_runner,
-        runs_publisher=runs_publisher,
-    )
+    """Get a singleton run data manager to keep track of current/historical run data."""
+    run_data_manager = _run_data_manager_accessor.get_from(app_state)
+
+    if run_data_manager is None:
+        run_data_manager = RunDataManager(
+            run_orchestrator_store=run_orchestrator_store,
+            run_store=run_store,
+            error_recovery_setting_store=error_recovery_setting_store,
+            task_runner=task_runner,
+            runs_publisher=runs_publisher,
+        )
+        _run_data_manager_accessor.set_on(app_state, run_data_manager)
+
+    return run_data_manager
 
 
 async def get_run_auto_deleter(

robot-server/robot_server/runs/router/__init__.py

@@ -5,12 +5,14 @@
 from .commands_router import commands_router
 from .actions_router import actions_router
 from .labware_router import labware_router
+from .error_recovery_policy_router import error_recovery_policy_router
 
 runs_router = APIRouter()
 
 runs_router.include_router(base_router)
 runs_router.include_router(commands_router)
 runs_router.include_router(actions_router)
 runs_router.include_router(labware_router)
+runs_router.include_router(error_recovery_policy_router)
 
 __all__ = ["runs_router"]