diff --git a/api/server/controllers/agents/__tests__/openai.spec.js b/api/server/controllers/agents/__tests__/openai.spec.js index 2a3bde6ec6..5d51069f26 100644 --- a/api/server/controllers/agents/__tests__/openai.spec.js +++ b/api/server/controllers/agents/__tests__/openai.spec.js @@ -1281,6 +1281,7 @@ describe('OpenAIChatCompletionController', () => { const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); + req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true }; await OpenAIChatCompletionController(req, res); const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1); @@ -1308,6 +1309,7 @@ describe('OpenAIChatCompletionController', () => { ); const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0]; + expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true); await toolExecuteOptions.loadTools(['file_search'], 'agent-123'); expect(loadToolsForExecution).toHaveBeenLastCalledWith( expect.objectContaining({ diff --git a/api/server/controllers/agents/__tests__/responses.unit.spec.js b/api/server/controllers/agents/__tests__/responses.unit.spec.js index 0bbf27b8f5..7ab3cef884 100644 --- a/api/server/controllers/agents/__tests__/responses.unit.spec.js +++ b/api/server/controllers/agents/__tests__/responses.unit.spec.js @@ -1872,6 +1872,7 @@ describe('createResponse controller', () => { const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService'); const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions'); + req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true }; await createResponse(req, res); const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1); @@ -1899,6 +1900,7 @@ describe('createResponse controller', () => { ); const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0]; + expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true); await toolExecuteOptions.loadTools(['file_search'], 'agent-123'); expect(loadToolsForExecution).toHaveBeenLastCalledWith( expect.objectContaining({ diff --git a/api/server/controllers/agents/openai.js b/api/server/controllers/agents/openai.js index 12cd8f126e..629fa8a539 100644 --- a/api/server/controllers/agents/openai.js +++ b/api/server/controllers/agents/openai.js @@ -466,6 +466,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents]; const allowedProviders = new Set(agentsEConfig?.allowedProviders); + const ordinaryToolCancellationEnabled = + agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true; // Create tool loader const loadTools = createToolLoader({ req, res, signal: execution.signal }); @@ -801,6 +803,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => { agent never gains sandbox access even if the admin enabled the capability globally. */ const toolExecuteOptions = { + ordinaryToolCancellation: ordinaryToolCancellationEnabled, provisionFiles: createProvisionFilesCallback({ req, agentToolContexts, diff --git a/api/server/controllers/agents/responses.js b/api/server/controllers/agents/responses.js index 02b7ea8dbd..bb16156cd6 100644 --- a/api/server/controllers/agents/responses.js +++ b/api/server/controllers/agents/responses.js @@ -703,6 +703,8 @@ const executeResponse = async (envelope, { req, res }) => { codeWorkspaces: request.code_workspaces ?? req.resolvedConversation?.codeWorkspaces, }); const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents]; + const ordinaryToolCancellationEnabled = + agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true; const previousMessages = request.previous_response_id ? await loadPreviousMessages(request.previous_response_id, principal.userId) : []; @@ -1144,6 +1146,7 @@ const executeResponse = async (envelope, { req, res }) => { // Create tool execute options for event-driven tool execution const toolExecuteOptions = { + ordinaryToolCancellation: ordinaryToolCancellationEnabled, provisionFiles: createProvisionFilesCallback({ req, agentToolContexts, @@ -1372,6 +1375,7 @@ const executeResponse = async (envelope, { req, res }) => { }); const toolExecuteOptions = { + ordinaryToolCancellation: ordinaryToolCancellationEnabled, provisionFiles: createProvisionFilesCallback({ req, agentToolContexts, diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index 371f9fa461..0a412070e3 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -183,6 +183,9 @@ const initializeClient = async ({ const completionWakeupsEnabled = backgroundCompletionWakeupsEnabled( appConfig?.endpoints?.[EModelEndpoint.agents], ); + const ordinaryToolCancellationEnabled = + appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.ordinaryToolCancellation === + true; /** The normal controller resolves this once for timestamp anchoring. Reuse * that trusted document for child-thread execution policy; resume and direct * callers fall back to the same owner-scoped lookup. */ @@ -407,6 +410,7 @@ const initializeClient = async ({ // is composed with this authoritative job signal by the handler. runSignal: signal, foregroundRunId: runtimeRequestBody?.messageId, + ordinaryToolCancellation: ordinaryToolCancellationEnabled, loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => { const ctx = agentToolContexts.get(agentId) ?? {}; logger.debug(`[ON_TOOL_EXECUTE] ctx found: ${!!ctx.userMCPAuthMap}, agent: ${ctx.agent?.id}`); diff --git a/api/server/services/Endpoints/agents/initialize.spec.js b/api/server/services/Endpoints/agents/initialize.spec.js index 52008d2198..ade30c146f 100644 --- a/api/server/services/Endpoints/agents/initialize.spec.js +++ b/api/server/services/Endpoints/agents/initialize.spec.js @@ -321,6 +321,31 @@ describe('initializeClient — processAgent ACL gate', () => { expect(capturedToolExecuteOptions.foregroundRunId).toBe('response-1'); }); + it('threads the deployment ordinary-tool cancellation gate into execution', async () => { + mockInitializeAgent.mockResolvedValue(makePrimaryConfig([])); + const disabledReq = makeReq(); + + await initializeClient({ + req: disabledReq, + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }); + expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(false); + + const enabledReq = makeReq(); + enabledReq.config.endpoints.agents = { + backgroundTasks: { ordinaryToolCancellation: true }, + }; + await initializeClient({ + req: enabledReq, + res: {}, + signal: new AbortController().signal, + endpointOption: makeEndpointOption(), + }); + expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(true); + }); + it('propagates an expected-MCP-tools failure from the runtime tool loader', async () => { const toolError = Object.assign(new Error('Expected MCP tools are unavailable'), { code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE', diff --git a/client/src/components/Chat/Messages/Content/Part.tsx b/client/src/components/Chat/Messages/Content/Part.tsx index 5454b17dc1..f1e9030779 100644 --- a/client/src/components/Chat/Messages/Content/Part.tsx +++ b/client/src/components/Chat/Messages/Content/Part.tsx @@ -209,6 +209,7 @@ const Part = memo(function Part({ runStepStatus={toolCall.runStepStatus} runStepDurationMs={toolCall.runStepDurationMs} backgrounded={toolCall.backgrounded} + backgroundCancelled={toolCall.backgroundTask?.cancelled === true} attachments={attachments} commandField="code" hideAttachments={hideAttachments} @@ -228,6 +229,7 @@ const Part = memo(function Part({ runStepStatus={toolCall.runStepStatus} runStepDurationMs={toolCall.runStepDurationMs} backgrounded={toolCall.backgrounded} + backgroundCancelled={toolCall.backgroundTask?.cancelled === true} output={toolCall.output ?? ''} initialProgress={toolCall.progress ?? 0.1} args={toolCall.args} @@ -361,6 +363,7 @@ const Part = memo(function Part({ runStepStatus={toolCall.runStepStatus} runStepDurationMs={toolCall.runStepDurationMs} backgrounded={toolCall.backgrounded} + backgroundCancelled={toolCall.backgroundTask?.cancelled === true} attachments={attachments} hideAttachments={hideAttachments} onExpand={onToolExpand} @@ -410,7 +413,9 @@ const Part = memo(function Part({ isLast={isLast} hideAttachments={hideAttachments} onExpand={onToolExpand} - runStepStatus={toolCall.runStepStatus} + runStepStatus={ + toolCall.backgroundTask?.cancelled === true ? 'cancelled' : toolCall.runStepStatus + } runStepDurationMs={toolCall.runStepDurationMs} /> ); diff --git a/client/src/components/Chat/Messages/Content/Parts/BashCall.tsx b/client/src/components/Chat/Messages/Content/Parts/BashCall.tsx index 11156abc9a..dd1c59fb80 100644 --- a/client/src/components/Chat/Messages/Content/Parts/BashCall.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/BashCall.tsx @@ -25,6 +25,7 @@ export default function BashCall({ runStepStatus, runStepDurationMs, backgrounded, + backgroundCancelled = false, initialProgress = 0.1, args, output = '', @@ -39,6 +40,7 @@ export default function BashCall({ runStepStatus?: PartMetadata['runStepStatus']; runStepDurationMs?: PartMetadata['runStepDurationMs']; backgrounded?: PartMetadata['backgrounded']; + backgroundCancelled?: boolean; args?: string | Record; output?: string; attachments?: TAttachment[]; @@ -67,6 +69,8 @@ export default function BashCall({ [attachments, toolCallId], ); const backgroundFailed = backgroundHandle != null && backgroundStatus === 'error'; + const cancelledInBackground = + backgroundCancelled || (backgroundHandle != null && backgroundStatus === 'cancelled'); const backgroundFinishedText = backgroundHandle ? localize( backgroundStatus != null || (fileAttachments?.length ?? 0) > 0 @@ -83,6 +87,7 @@ export default function BashCall({ onExpand, runStepStatus, extraError: backgroundFailed, + extraCancelled: cancelledInBackground, }); const highlighted = useLazyHighlight(command || undefined, 'bash'); diff --git a/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx b/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx index f969df1897..35678fb929 100644 --- a/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/ExecuteCode.tsx @@ -63,6 +63,7 @@ export default function ExecuteCode({ runStepStatus, runStepDurationMs, backgrounded, + backgroundCancelled = false, initialProgress = 0.1, args, output = '', @@ -76,6 +77,7 @@ export default function ExecuteCode({ runStepStatus?: PartMetadata['runStepStatus']; runStepDurationMs?: PartMetadata['runStepDurationMs']; backgrounded?: PartMetadata['backgrounded']; + backgroundCancelled?: boolean; args?: string | Record; output?: string; attachments?: TAttachment[]; @@ -105,6 +107,8 @@ export default function ExecuteCode({ [attachments, toolCallId], ); const backgroundFailed = backgroundHandle != null && backgroundStatus === 'error'; + const cancelledInBackground = + backgroundCancelled || (backgroundHandle != null && backgroundStatus === 'cancelled'); const backgroundFinishedText = backgroundHandle ? localize( backgroundStatus != null || (fileAttachments?.length ?? 0) > 0 @@ -121,6 +125,7 @@ export default function ExecuteCode({ onExpand, runStepStatus, extraError: backgroundFailed, + extraCancelled: cancelledInBackground, }); const highlighted = useLazyHighlight(code, lang); diff --git a/client/src/components/Chat/Messages/Content/Parts/__tests__/BashCall.test.tsx b/client/src/components/Chat/Messages/Content/Parts/__tests__/BashCall.test.tsx index 7775c80e02..d12943c5d4 100644 --- a/client/src/components/Chat/Messages/Content/Parts/__tests__/BashCall.test.tsx +++ b/client/src/components/Chat/Messages/Content/Parts/__tests__/BashCall.test.tsx @@ -261,6 +261,36 @@ describe('BashCall backgrounded calls', () => { expect(screen.getByTestId('progress-text')).toHaveTextContent('tool failed'); }); + it('surfaces a live cancelled marker as cancellation rather than failure', () => { + renderBackgrounded([ + { + type: 'background_task_status', + file_id: 'bg-tc-1', + toolCallId: 'tc-1', + status: 'cancelled', + }, + ]); + expect(screen.getByTestId('progress-text')).toHaveTextContent('Cancelled'); + expect(screen.getByTestId('progress-text')).not.toHaveTextContent('tool failed'); + }); + + it('restores a persisted background cancellation as cancellation', () => { + render( + + + , + ); + expect(screen.getByTestId('progress-text')).toHaveTextContent('Cancelled'); + expect(screen.getByTestId('progress-text')).not.toHaveTextContent('tool failed'); + }); + it('renders real stdout normally after the background result patches the output', () => { render( diff --git a/client/src/components/Chat/Messages/Content/Parts/__tests__/handle.test.ts b/client/src/components/Chat/Messages/Content/Parts/__tests__/handle.test.ts index 366ab7a2ad..539908f26d 100644 --- a/client/src/components/Chat/Messages/Content/Parts/__tests__/handle.test.ts +++ b/client/src/components/Chat/Messages/Content/Parts/__tests__/handle.test.ts @@ -75,6 +75,16 @@ describe('parseBackgroundHandle', () => { expect(fileAttachments).toHaveLength(0); }); + it('preserves the cancelled terminal status from an owned marker', () => { + const marker = { + type: 'background_task_status', + file_id: 'bg-tc-mine', + toolCallId: 'tc-mine', + status: 'cancelled', + } as never; + expect(splitBackgroundAttachments([marker], 'tc-mine').backgroundStatus).toBe('cancelled'); + }); + it('returns null for oversized payloads (real output, not a handle)', () => { const big = JSON.stringify({ background_task_id: 'task-1', diff --git a/client/src/components/Chat/Messages/Content/Parts/useToolCallState.ts b/client/src/components/Chat/Messages/Content/Parts/useToolCallState.ts index 18e0aae1cf..fc5d53bc03 100644 --- a/client/src/components/Chat/Messages/Content/Parts/useToolCallState.ts +++ b/client/src/components/Chat/Messages/Content/Parts/useToolCallState.ts @@ -37,6 +37,10 @@ export interface UseToolCallStateInput { * every other card instead of patching the result afterwards. */ extraError?: boolean; + /** A terminal verdict carried outside the dispatch run step, such as a + * cancelled ordinary background tool. Cancellation outranks error-shaped + * output so the card never relabels an intentional stop as failure. */ + extraCancelled?: boolean; } export default function useToolCallState({ @@ -47,6 +51,7 @@ export default function useToolCallState({ onExpand, runStepStatus, extraError = false, + extraCancelled = false, }: UseToolCallStateInput): ToolCallState { const autoExpand = useRecoilValue(store.autoExpandTools); const hasOutput = output.length > 0; @@ -86,7 +91,7 @@ export default function useToolCallState({ * for why the cancellation inference must not read the animated one. */ const phase = resolveToolCallPhase({ - runStepStatus, + runStepStatus: extraCancelled ? 'cancelled' : runStepStatus, displayProgress: rawProgress, reportedProgress: initialProgress, isSubmitting, diff --git a/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx b/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx index feb18a4f01..89aa5d3edf 100644 --- a/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx +++ b/client/src/components/Chat/Messages/Content/ToolCallGroup.tsx @@ -148,20 +148,23 @@ function getToolMeta( * reported no failed action beside a card showing failure. Correlated the * same way the child is, since provider tool-call ids repeat across agents * and execution steps in handoff responses. */ - const backgroundFailed = - parseBackgroundHandle(tc.output) != null && - splitBackgroundAttachments( - filterAttachmentsForPart( - attachmentsByToolCallId?.[tc.id ?? ''], - tc.agentId, - toolCall.stepId, - ), - tc.id, - ).backgroundStatus === 'error'; + const backgroundHandle = parseBackgroundHandle(tc.output); + const backgroundStatus = splitBackgroundAttachments( + filterAttachmentsForPart(attachmentsByToolCallId?.[tc.id ?? ''], tc.agentId, toolCall.stepId), + tc.id, + ).backgroundStatus; + const backgroundFailed = backgroundHandle != null && backgroundStatus === 'error'; + const backgroundCancelled = + tc.backgroundTask?.cancelled === true || + (backgroundHandle != null && backgroundStatus === 'cancelled'); return { name, iconName, - ...resolveOutcome(runStepStatus, completed, failedOutput || backgroundFailed), + ...resolveOutcome( + backgroundCancelled ? 'cancelled' : runStepStatus, + completed, + failedOutput || backgroundFailed, + ), }; } diff --git a/client/src/components/Chat/Messages/Content/__tests__/Part.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/Part.test.tsx index 7d833745d8..ecac2322e0 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/Part.test.tsx +++ b/client/src/components/Chat/Messages/Content/__tests__/Part.test.tsx @@ -57,7 +57,9 @@ jest.mock('../WebSearch', () => ({ jest.mock('../ToolCall', () => ({ __esModule: true, - default: () =>
, + default: ({ runStepStatus }: { runStepStatus?: string }) => ( +
+ ), })); jest.mock('../Image', () => ({ @@ -135,6 +137,21 @@ describe('Part tool renderer selection', () => { expect(screen.queryByTestId('tool-call')).not.toBeInTheDocument(); }); + it('renders a cancelled generic background tool as cancelled', () => { + const part = toolCallPart('search_mcp_docs') as Extract< + TMessageContentParts, + { type: typeof ContentTypes.TOOL_CALL } + >; + Object.assign(part[ContentTypes.TOOL_CALL], { + runStepStatus: 'failed', + backgroundTask: { cancelled: true }, + }); + + renderPart(part); + + expect(screen.getByTestId('tool-call')).toHaveAttribute('data-run-step-status', 'cancelled'); + }); + it('routes an unavailable reasoning marker to the marker renderer', () => { renderPart({ type: ContentTypes.THINK, diff --git a/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx index 745af4a8a9..e86e97ba82 100644 --- a/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx +++ b/client/src/components/Chat/Messages/Content/__tests__/ToolCallGroup.test.tsx @@ -961,6 +961,39 @@ describe('ToolCallGroup image hoisting', () => { ).toBeInTheDocument(); }); + it('counts a persisted ordinary background cancellation as cancelled', () => { + const cancelled = { + type: ContentTypes.TOOL_CALL, + [ContentTypes.TOOL_CALL]: { + id: 'c2', + name: 'bash_tool', + args: '{"command":"sleep 600"}', + output: 'Error: [bash_tool] tool call failed: Background task cancellation requested', + runStepStatus: 'completed', + backgrounded: true, + backgroundTask: { + version: 1, + taskId: 'background-task-1', + toolName: 'bash_tool', + status: 'error', + cancelled: true, + settledAt: new Date(), + }, + }, + } as unknown as TMessageContentParts; + + renderGroup({ + ...baseProps, + parts: [ + { part: makePart('c1', 'created', 'create_file'), idx: 0 }, + { part: cancelled, idx: 1 }, + ], + lastContentIdx: 1, + }); + + expect(screen.getByRole('button', { name: /· 1 cancelled$/ })).toBeInTheDocument(); + }); + it('labels a homogeneous ask_user_question group as its own category', () => { renderGroup({ ...baseProps, diff --git a/librechat.example.yaml b/librechat.example.yaml index 940e52ef41..b502457ac0 100644 --- a/librechat.example.yaml +++ b/librechat.example.yaml @@ -652,6 +652,8 @@ endpoints: # # Enabled by default; set false to keep background tools and subagents poll-only. # backgroundTasks: # completionWakeups: true + # # Allow owners to cooperatively cancel ordinary background tools, including BYOM Bash. + # ordinaryToolCancellation: false # # "run_in_background" makes Code Interpreter tools eligible by default and enables per-tool MCP opt-in. # # "tool_intents" enables live model-written labels for native tools and opted-in MCP tools. # # (optional) Require user approval before matching tool calls. Disabled by default. diff --git a/packages/api/src/agents/background.spec.ts b/packages/api/src/agents/background.spec.ts index b50969d24c..fa1303f465 100644 --- a/packages/api/src/agents/background.spec.ts +++ b/packages/api/src/agents/background.spec.ts @@ -675,6 +675,120 @@ describe('selection policy at injection time', () => { }); describe('BackgroundTaskRegistryClass', () => { + it('requests cancellation idempotently without settling or releasing running capacity', () => { + const registry = new BackgroundTaskRegistryClass(); + const requestCancellation = jest.fn(); + const created = registry.create({ + userId: 'cancel-owner', + conversationId: 'cancel-conversation', + toolCallId: 'cancel-call', + toolName: 'bash_tool', + requestCancellation, + }); + if ('atCapacity' in created) { + throw new Error('unexpected capacity'); + } + + expect( + registry.requestCancellation('other-owner', 'cancel-conversation', created.task.id), + ).toEqual({ status: 'not_found' }); + expect( + registry.requestCancellation('cancel-owner', 'other-conversation', created.task.id), + ).toEqual({ status: 'not_found' }); + expect( + registry.requestCancellation('cancel-owner', 'cancel-conversation', created.task.id).status, + ).toBe('requested'); + expect( + registry.requestCancellation('cancel-owner', 'cancel-conversation', created.task.id).status, + ).toBe('already_requested'); + expect(requestCancellation).toHaveBeenCalledTimes(1); + expect(created.task).toMatchObject({ + status: 'running', + cancellationRequestedAt: expect.any(Number), + }); + + for (let index = 0; index < 9; index += 1) { + const admitted = registry.create({ + userId: 'cancel-owner', + conversationId: 'cancel-conversation', + toolCallId: `other-call-${index}`, + toolName: 'background-tool', + }); + expect('atCapacity' in admitted).toBe(false); + } + expect( + registry.create({ + userId: 'cancel-owner', + conversationId: 'cancel-conversation', + toolCallId: 'blocked-until-settlement', + toolName: 'background-tool', + }), + ).toEqual({ atCapacity: true, scope: 'conversation_running' }); + + registry.cancel( + 'cancel-owner', + 'cancel-conversation', + created.task.id, + 'Background task cancellation requested', + ); + expect(created.task.status).toBe('cancelled'); + expect( + registry.create({ + userId: 'cancel-owner', + conversationId: 'cancel-conversation', + toolCallId: 'admitted-after-settlement', + toolName: 'background-tool', + }), + ).toMatchObject({ isNew: true }); + expect( + registry.requestCancellation('cancel-owner', 'cancel-conversation', created.task.id).status, + ).toBe('settled'); + }); + + it('lets actual completion win when an abort-resistant invocation settles successfully', () => { + const registry = new BackgroundTaskRegistryClass(); + const created = registry.create({ + userId: 'race-owner', + conversationId: 'race-conversation', + toolCallId: 'race-call', + toolName: 'mutation', + requestCancellation: jest.fn(), + }); + if ('atCapacity' in created) { + throw new Error('unexpected capacity'); + } + + registry.requestCancellation('race-owner', 'race-conversation', created.task.id); + registry.complete('race-owner', 'race-conversation', created.task.id, { + content: 'completed despite cancellation request', + }); + registry.cancel('race-owner', 'race-conversation', created.task.id, 'cancelled too late'); + + expect(created.task).toMatchObject({ + status: 'completed', + result: 'completed despite cancellation request', + }); + }); + + it('does not record a manual cancellation when another abort source already won', () => { + const registry = new BackgroundTaskRegistryClass(); + const created = registry.create({ + userId: 'timeout-owner', + conversationId: 'timeout-conversation', + toolCallId: 'timeout-call', + toolName: 'mutation', + requestCancellation: () => false, + }); + if ('atCapacity' in created) { + throw new Error('unexpected capacity'); + } + + expect( + registry.requestCancellation('timeout-owner', 'timeout-conversation', created.task.id), + ).toMatchObject({ status: 'unavailable' }); + expect(created.task.cancellationRequestedAt).toBeUndefined(); + }); + it('creates, completes, and reads a task', () => { const registry = new BackgroundTaskRegistryClass(); const created = registry.create({ @@ -1970,6 +2084,10 @@ describe('getBackgroundCodeDelivery (singleton)', () => { messageId: 'dispatch-msg', result: 'stdout', attachments: [{ file_id: 'f1' }], + backgroundTask: expect.objectContaining({ + taskId: created.task.id, + status: 'completed', + }), }), ); /** Not one-shot: a later poll can still re-emit / re-anchor. */ @@ -2017,6 +2135,84 @@ describe('runCheckBackgroundTask (singleton)', () => { ); }); + it('keeps ordinary cancellation disabled unless the deployment opts in', async () => { + const requestCancellation = jest.fn(); + const created = backgroundTaskRegistry.create({ + userId: 'disabled-cancel-user', + conversationId: 'disabled-cancel-conversation', + toolCallId: 'disabled-cancel-call', + toolName: 'bash_tool', + requestCancellation, + }); + if ('atCapacity' in created) { + throw new Error('unexpected capacity'); + } + + const result = JSON.parse( + await runCheckBackgroundTask({ + userId: 'disabled-cancel-user', + conversationId: 'disabled-cancel-conversation', + args: { background_task_id: created.task.id, action: 'cancel' }, + }), + ); + expect(result).toMatchObject({ status: 'invalid', background_task_id: created.task.id }); + expect(requestCancellation).not.toHaveBeenCalled(); + backgroundTaskRegistry.fail( + 'disabled-cancel-user', + 'disabled-cancel-conversation', + created.task.id, + 'test cleanup', + ); + }); + + it('uses normal durable claim arbitration when cancellation loses the settlement race', async () => { + const created = backgroundTaskRegistry.create({ + userId: 'cancel-race-user', + conversationId: 'cancel-race-conversation', + toolCallId: 'cancel-race-call', + toolName: 'bash_tool', + messageId: 'cancel-race-message', + requestCancellation: jest.fn(), + }); + if ('atCapacity' in created) { + throw new Error('unexpected capacity'); + } + backgroundTaskRegistry.markCompletionWakeup( + 'cancel-race-user', + 'cancel-race-conversation', + created.task.id, + { + renew: jest.fn(async () => true), + retire: jest.fn(async () => true), + }, + ); + backgroundTaskRegistry.complete( + 'cancel-race-user', + 'cancel-race-conversation', + created.task.id, + { content: 'settled result' }, + ); + const claimBackgroundToolResult = jest.fn(async () => ({ + status: 'claimed' as const, + claim: { kind: 'wakeup' as const, claimId: 'automatic-delivery' }, + })); + + const result = JSON.parse( + await runCheckBackgroundTask({ + userId: 'cancel-race-user', + conversationId: 'cancel-race-conversation', + args: { background_task_id: created.task.id, action: 'cancel' }, + ordinaryToolCancellation: true, + claimBackgroundToolResult, + }), + ); + expect(result).toMatchObject({ + status: 'delivery_scheduled', + background_task_id: created.task.id, + }); + expect(claimBackgroundToolResult).toHaveBeenCalledTimes(1); + }); + it('rejects an oversized task id before local or cross-replica lookup', async () => { const store = Object.assign(new InMemorySubagentTaskStore(), { claimTask: jest.fn(), diff --git a/packages/api/src/agents/background.ts b/packages/api/src/agents/background.ts index 8a5c50a0b0..de752c68a0 100644 --- a/packages/api/src/agents/background.ts +++ b/packages/api/src/agents/background.ts @@ -53,6 +53,7 @@ import type { BackgroundToolResultRecord, } from '@librechat/data-schemas'; import type { AgentToolOptions } from 'librechat-data-provider'; +import type { BackgroundToolResultState } from './harvest'; import type { CapabilityToolNames } from './selection'; import { BACKGROUND_TASK_TIMEOUT_MS, @@ -353,7 +354,7 @@ export function stripBackgroundFromToolRegistry( const CHECK_BACKGROUND_TASK_DESCRIPTION = `Check, control, and retrieve tool or subagent tasks previously dispatched in the background (with run_in_background: true). -Provide a background_task_id to poll one task; omit it to list every background task in this thread. A task is only finished when its status is "completed", "error", or "cancelled" — never assume completion without polling. Results are not pushed to you; you must call this tool to collect them. Subagent tasks additionally accept steer, queue, interrupt, cancel, and cancel_message actions while running. Live subagent controls route across API replicas but do not survive a restart of the process that owns the executor. A completed subagent thread may be continued later through the subagent tool's durable thread id.`; +Provide a background_task_id to poll one task; omit it to list every background task in this thread. A task is only finished when its status is "completed", "error", or "cancelled" — never assume completion without polling. Results are not pushed to you; you must call this tool to collect them. The cancel action applies to subagents and to ordinary tools when enabled by the deployment; steer, queue, interrupt, and cancel_message apply only to subagents. Live controls route to process-local executors and do not survive an owning-process restart. A completed subagent thread may be continued later through the subagent tool's durable thread id.`; const CHECK_BACKGROUND_TASK_WAKEUP_DESCRIPTION = `Check, control, and retrieve tool or subagent tasks previously dispatched in the background (with run_in_background: true). @@ -399,7 +400,8 @@ const CHECK_BACKGROUND_TASK_PARAMETERS = Object.freeze(); private readonly retainedUsage = new WeakMap(); + /** Live invocation controls are intentionally process-local and are never + * exposed through task snapshots or durable receipts. */ + private readonly cancellationRequests = new WeakMap boolean | void>(); private lastGlobalSweepAt = 0; private key(userId: string, conversationId: string): string { @@ -1213,6 +1222,10 @@ export class BackgroundTaskRegistryClass { harvestStarted?: boolean; liveArtifactPollRequired?: boolean; capacityPermit?: BackgroundTaskCapacityPermit; + /** Abort the already-created invocation signal. Ownership is enforced by + * requestCancellation before this closure can be reached. */ + /** Returns false when another abort source already won the race. */ + requestCancellation?: () => boolean | void; }): { task: BackgroundTask; isNew: boolean } | BackgroundTaskCapacityRejection { const now = Date.now(); this.sweep(now); @@ -1286,9 +1299,42 @@ export class BackgroundTaskRegistryClass { }; bucket.tasks.set(task.id, task); bucket.byToolCall.set(dedupeKey, task.id); + if (params.requestCancellation != null) { + this.cancellationRequests.set(task, params.requestCancellation); + } return { task, isNew: true }; } + requestCancellation( + userId: string, + conversationId: string, + taskId: string, + ): + | { status: 'not_found' } + | { status: 'unavailable'; task: BackgroundTask } + | { status: 'requested' | 'already_requested' | 'settled'; task: BackgroundTask } { + const task = this.get(userId, conversationId, taskId); + if (task == null) { + return { status: 'not_found' }; + } + if (task.status !== 'running') { + return { status: 'settled', task }; + } + if (task.cancellationRequestedAt != null) { + return { status: 'already_requested', task }; + } + const request = this.cancellationRequests.get(task); + if (request == null) { + return { status: 'unavailable', task }; + } + if (request() === false) { + return { status: 'unavailable', task }; + } + task.cancellationRequestedAt = Date.now(); + task.updatedAt = task.cancellationRequestedAt; + return { status: 'requested', task }; + } + private update( userId: string, conversationId: string, @@ -1338,6 +1384,7 @@ export class BackgroundTaskRegistryClass { artifactDelivered: false, }); if (updated) { + this.cancellationRequests.delete(task); this.updatePayloadUsage(task, { result: retainedContent?.length ?? 0, artifact: artifactChars, @@ -1508,6 +1555,40 @@ export class BackgroundTaskRegistryClass { ...(options?.harvestStarted === true ? { harvestStarted: true, harvestPending: true } : {}), }); if (updated) { + this.cancellationRequests.delete(task); + this.retainedUsage.set(task, { + result: 0, + artifact: 0, + attachments: 0, + error: retainedError?.length ?? 0, + }); + } + } + + cancel( + userId: string, + conversationId: string, + taskId: string, + error: string, + options?: { harvestStarted?: boolean }, + ): void { + const storedError = truncateMiddle(error, MAX_RESULT_CHARS); + const task = this.buckets.get(this.key(userId, conversationId))?.tasks.get(taskId); + if (task == null || task.status !== 'running' || task.artifactBlocked === true) { + return; + } + const hasRetainedCapacity = this.makeRetainedRoom(userId, task, storedError.length); + const retainedError = hasRetainedCapacity ? storedError : undefined; + const updated = this.update(userId, conversationId, taskId, { + status: 'cancelled', + error: retainedError, + result: undefined, + artifact: undefined, + attachments: undefined, + ...(options?.harvestStarted === true ? { harvestStarted: true, harvestPending: true } : {}), + }); + if (updated) { + this.cancellationRequests.delete(task); this.retainedUsage.set(task, { result: 0, artifact: 0, @@ -1749,6 +1830,7 @@ interface SerializedBackgroundTask { status: BackgroundTaskStatus; /** Coarse 0..1: no intermediate progress exists, only running vs settled. */ progress: number; + cancellation_requested?: boolean; result?: string; result_available?: boolean; result_chars?: number; @@ -1795,6 +1877,9 @@ function serializeTask( tool: task.toolName, status: task.status, progress: task.status === 'running' ? 0 : 1, + ...(task.status === 'running' && task.cancellationRequestedAt != null + ? { cancellation_requested: true } + : {}), ...resultFields(task, includeResult), ...taskNote(task), ...(task.error !== undefined ? { error: task.error } : {}), @@ -1982,6 +2067,8 @@ export async function runCheckBackgroundTask(params: { allowUnfinished?: boolean; }) => Promise; recoverDeadBackgroundToolClaim?: BackgroundToolDeadClaimRecovery; + /** Trusted deployment policy. Defaults false for backward compatibility. */ + ordinaryToolCancellation?: boolean; }): Promise { const { userId, conversationId } = params; const args = coerceArgsObject(params.args) ?? {}; @@ -2000,11 +2087,48 @@ export async function runCheckBackgroundTask(params: { const task = backgroundTaskRegistry.get(userId, conversationId, taskId); if (task != null) { if (action !== 'poll') { - return JSON.stringify({ - status: 'invalid', - background_task_id: taskId, - message: 'Control actions are supported only for subagent tasks.', - }); + if (action === 'cancel') { + if (params.ordinaryToolCancellation !== true) { + return JSON.stringify({ + status: 'invalid', + background_task_id: taskId, + message: 'Cancellation is not enabled for ordinary background tools.', + }); + } + const cancellation = backgroundTaskRegistry.requestCancellation( + userId, + conversationId, + taskId, + ); + if (cancellation.status === 'settled') { + /** Settlement won the race. Fall through to the ordinary poll path + * so durable result-claim arbitration still elects one consumer. */ + } else if ( + cancellation.status === 'requested' || + cancellation.status === 'already_requested' + ) { + return JSON.stringify({ + ...serializeTask(cancellation.task, { includeResult: false }), + status: 'cancellation_requested', + cancellation_requested: true, + message: + 'Cancellation was requested. The task remains active until its executor settles; poll again for a terminal result.', + }); + } else { + return JSON.stringify({ + status: 'unavailable', + background_task_id: taskId, + message: + 'This server no longer owns a live cancellation handle for the task. Poll for its outcome; do not assume execution stopped.', + }); + } + } else { + return JSON.stringify({ + status: 'invalid', + background_task_id: taskId, + message: 'This control action is supported only for subagent tasks.', + }); + } } if (task.status !== 'running') { if ( @@ -2545,6 +2669,7 @@ export function getBackgroundCodeDelivery(params: { result?: string; error?: string; attachments?: unknown[]; + backgroundTask?: BackgroundToolResultState; } | undefined { const rawId = coerceArgsObject(params.args)?.background_task_id; @@ -2572,6 +2697,30 @@ export function getBackgroundCodeDelivery(params: { result: task.result, error: task.error, attachments: task.attachments, + ...(task.status === 'running' + ? {} + : { + backgroundTask: { + taskId: task.id, + toolName: task.toolName, + status: task.status === 'cancelled' ? 'error' : task.status, + ...(task.status === 'cancelled' ? { cancelled: true } : {}), + settledAt: new Date(task.updatedAt), + ...(task.completionWakeup === true ? { completionWakeup: true } : {}), + ...(task.resultClaim == null + ? {} + : { + resultClaim: { + kind: task.resultClaim.kind, + claimId: task.resultClaim.claimId, + claimedAt: new Date(task.resultClaim.claimedAt), + ...(task.resultClaim.generationId == null + ? {} + : { generationId: task.resultClaim.generationId }), + }, + }), + }, + }), }; } diff --git a/packages/api/src/agents/backgroundCompletionWakeup.ts b/packages/api/src/agents/backgroundCompletionWakeup.ts index 03fddfefeb..bdb2686940 100644 --- a/packages/api/src/agents/backgroundCompletionWakeup.ts +++ b/packages/api/src/agents/backgroundCompletionWakeup.ts @@ -218,7 +218,7 @@ function buildWakeupInput( taskId: string; toolCallId: string; toolName: string; - status: 'completed' | 'error'; + status: 'completed' | 'error' | 'cancelled'; output: string; }>, ): string { diff --git a/packages/api/src/agents/handlers.background.spec.ts b/packages/api/src/agents/handlers.background.spec.ts index 0e613bc689..c1f972df15 100644 --- a/packages/api/src/agents/handlers.background.spec.ts +++ b/packages/api/src/agents/handlers.background.spec.ts @@ -2,6 +2,7 @@ import { z } from 'zod'; import { logger } from '@librechat/data-schemas'; import type { StructuredToolInterface } from '@librechat/agents/langchain/tools'; import type { FiltersConfig } from 'librechat-data-provider'; +import type { ToolExecuteOptions } from './handlers'; import { backgroundTaskRegistry, runCheckBackgroundTask, @@ -618,6 +619,95 @@ describe('createToolExecuteHandler — background tool calls', () => { } }); + it('preserves timeout provenance when cancellation is requested after the deadline', async () => { + jest.useFakeTimers({ doNotFake: ['setImmediate'] }); + try { + let rejectInvocation: ((error: unknown) => void) | undefined; + let invocationSignal: AbortSignal | undefined; + const tool = { + name: 'execute_code', + description: 'settles after acknowledging an abort', + schema: z.object({ lang: z.string(), code: z.string() }), + invoke: jest.fn( + (_input: unknown, config?: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + invocationSignal = config?.signal; + rejectInvocation = reject; + }), + ), + } as unknown as StructuredToolInterface; + const persistBackgroundCodeResult = jest.fn( + async ( + _params: Parameters>[0], + ) => ({ attachments: [] }), + ); + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [tool] }), + ordinaryToolCancellation: true, + persistBackgroundCodeResult, + backgroundToolCompletion: { + preregister: async () => ({ + renew: jest.fn(async () => true), + retire: jest.fn(async () => true), + }), + persist: jest.fn(async () => true), + claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })), + }, + }); + const configurable = buildConfig([tool.name]); + const metadata = { thread_id: 'late_cancel_conversation', run_id: 'late-cancel-run' }; + const [dispatch] = await runBatch(handler, { + toolCalls: [ + { + id: 'call_late_cancel', + name: 'execute_code', + args: { lang: 'python', code: 'while True: pass', run_in_background: true }, + }, + ], + agentId: 'agent_late_cancel', + configurable, + metadata, + }); + const taskId = JSON.parse(dispatch.content).background_task_id as string; + + jest.advanceTimersByTime(31 * 60 * 1000); + await flushMicrotasks(); + const [cancellation] = await runBatch(handler, { + toolCalls: [ + { + id: 'call_late_cancel_control', + name: CHECK_BACKGROUND_TASK_NAME, + args: { background_task_id: taskId, action: 'cancel' }, + }, + ], + agentId: 'agent_late_cancel', + configurable, + metadata, + }); + expect(JSON.parse(cancellation.content)).toMatchObject({ + status: 'unavailable', + background_task_id: taskId, + }); + + rejectInvocation?.( + invocationSignal?.reason ?? new DOMException('Background task timed out', 'AbortError'), + ); + await flushMicrotasks(); + await flushMicrotasks(); + + const persisted = + persistBackgroundCodeResult.mock.calls[ + persistBackgroundCodeResult.mock.calls.length - 1 + ]?.[0]; + expect(persisted).toEqual( + expect.objectContaining({ output: expect.stringContaining('Background task timed out') }), + ); + expect(persisted?.backgroundTask?.cancelled).not.toBe(true); + } finally { + jest.useRealTimers(); + } + }); + it('retires the automatic wakeup after an abort-resistant invocation exceeds its grace period', async () => { jest.useFakeTimers({ doNotFake: ['setImmediate'] }); try { @@ -977,7 +1067,7 @@ describe('createToolExecuteHandler — background tool calls', () => { expect(eventActorDetachedAction.wake).toHaveBeenCalledTimes(1); }); - it('records an aborted detached expected action as cancelled', async () => { + it('does not claim an unrequested provider abort was owner cancellation', async () => { const abortError = Object.assign(new Error('operation aborted'), { name: 'AbortError' }); const tool = { name: 'submit_move_mcp_chess', @@ -1018,7 +1108,7 @@ describe('createToolExecuteHandler — background tool calls', () => { await flushMicrotasks(); expect(eventActorDetachedAction.settle).toHaveBeenCalledWith( - expect.objectContaining({ status: 'cancelled', error: 'operation aborted' }), + expect.objectContaining({ status: 'failed', error: 'operation aborted' }), ); }); @@ -1074,6 +1164,154 @@ describe('createToolExecuteHandler — background tool calls', () => { expect(polled.result).toContain('RESULT for librechat'); }); + it('cancels a running background Bash mutation, reports settlement truthfully, and reuses the workspace', async () => { + let delayedWrites = 0; + let invocations = 0; + const bashTool = { + name: 'bash_tool', + description: 'runs a workspace command', + schema: z.object({ command: z.string() }), + invoke: jest.fn( + async ( + input: { command: string }, + config?: { signal?: AbortSignal }, + ): Promise<{ content: string }> => { + invocations += 1; + if (input.command === 'pwd') { + return { content: '/workspace' }; + } + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + delayedWrites += 1; + resolve({ content: 'late mutation completed' }); + }, 50); + config?.signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + reject( + config.signal?.reason ?? + new DOMException('Background task cancellation requested', 'AbortError'), + ); + }, + { once: true }, + ); + }); + }, + ), + } as unknown as StructuredToolInterface; + const persistBackgroundCodeResult = jest.fn(async () => ({ attachments: [] })); + const handler = createToolExecuteHandler({ + loadTools: async () => ({ loadedTools: [bashTool] }), + ordinaryToolCancellation: true, + persistBackgroundCodeResult, + }); + const configurable = buildConfig(['bash_tool']); + const metadata = { thread_id: 'cancel_bash_conversation', run_id: 'cancel-bash-run' }; + + const dispatch = await runBatch(handler, { + toolCalls: [ + { + id: 'call_cancel_bash', + name: 'bash_tool', + stepId: 'step_cancel_bash', + args: { command: 'sleep-then-write', run_in_background: true }, + }, + ], + agentId: 'agent_cancel_bash', + configurable, + metadata, + }); + const taskId = JSON.parse(dispatch[0].content).background_task_id as string; + + const cancellation = JSON.parse( + ( + await runBatch(handler, { + toolCalls: [ + { + id: 'call_cancel_control', + name: CHECK_BACKGROUND_TASK_NAME, + args: { background_task_id: taskId, action: 'cancel' }, + }, + ], + agentId: 'agent_cancel_bash', + configurable, + metadata, + }) + )[0].content, + ); + expect(cancellation).toMatchObject({ + status: 'cancellation_requested', + cancellation_requested: true, + }); + + await flushMicrotasks(); + await flushMicrotasks(); + await new Promise((resolve) => setTimeout(resolve, 75)); + const terminal = JSON.parse( + ( + await runBatch(handler, { + toolCalls: [ + { + id: 'call_poll_cancelled_bash', + name: CHECK_BACKGROUND_TASK_NAME, + args: { background_task_id: taskId }, + }, + ], + agentId: 'agent_cancel_bash', + configurable, + metadata, + }) + )[0].content, + ); + expect(terminal).toMatchObject({ + status: 'cancelled', + background_task_id: taskId, + }); + expect(persistBackgroundCodeResult).toHaveBeenCalledWith( + expect.objectContaining({ + backgroundTask: expect.objectContaining({ taskId, status: 'error', cancelled: true }), + }), + ); + expect(persistBackgroundCodeResult).toHaveBeenCalledWith( + expect.objectContaining({ + reapply: true, + output: expect.stringContaining('cancellation requested'), + backgroundTask: expect.objectContaining({ + taskId, + status: 'error', + cancelled: true, + }), + }), + ); + expect(delayedWrites).toBe(0); + + const reused = await runBatch(handler, { + toolCalls: [ + { + id: 'call_reuse_bash', + name: 'bash_tool', + args: { command: 'pwd', run_in_background: true }, + }, + ], + agentId: 'agent_cancel_bash', + configurable, + metadata: { ...metadata, run_id: 'reuse-run' }, + }); + const reusedTaskId = JSON.parse(reused[0].content).background_task_id as string; + await flushMicrotasks(); + await flushMicrotasks(); + const reusedTerminal = JSON.parse( + await runCheckBackgroundTask({ + userId: 'exec_user', + conversationId: 'cancel_bash_conversation', + args: { background_task_id: reusedTaskId }, + }), + ); + expect(reusedTerminal).toMatchObject({ status: 'completed', result: '/workspace' }); + expect(invocations).toBe(2); + }); + it('blocks normalized arguments before registering or dispatching a background task', async () => { const protectedValue = 'PROTECTED-BACKGROUND'; const state = { calls: 0 } as { calls: number; lastInput?: Record }; diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts index d29d4cec86..5345067ae5 100644 --- a/packages/api/src/agents/handlers.ts +++ b/packages/api/src/agents/handlers.ts @@ -244,6 +244,8 @@ export interface ToolExecuteOptions { }>; /** Trusted detached-subagent task scope for polling and parent controls. */ subagentTasks?: SubagentTaskConfig; + /** Trusted deployment gate for cooperative ordinary-tool cancellation. */ + ordinaryToolCancellation?: boolean; /** Callback to process tool artifacts (code output files, file citations, etc.) */ toolEndCallback?: ToolEndCallback; /** Durable internal-completion adapter, present only for an Event Actor invocation. */ @@ -5209,6 +5211,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand emitAttachment, emitPtcProgress, subagentTasks, + ordinaryToolCancellation = false, provisionFiles, } = options; @@ -5516,12 +5519,24 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand }), }; } + const backgroundAbortController = new AbortController(); + let backgroundAbortSource: 'manual' | 'timeout' | undefined; const created = backgroundTaskRegistry.create({ ...(detachedReservation?.status === 'reserved' ? { taskId: detachedReservation.taskId } : {}), ...registration, ...(capacityPermit == null ? {} : { capacityPermit }), + requestCancellation: () => { + if (backgroundAbortSource != null || backgroundAbortController.signal.aborted) { + return false; + } + backgroundAbortSource = 'manual'; + backgroundAbortController.abort( + new DOMException('Background task cancellation requested', 'AbortError'), + ); + return true; + }, }); if ('atCapacity' in created) { if (detachedReservation?.status === 'reserved') { @@ -5587,7 +5602,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand const persistBackgroundResult = async (params: { output?: string; artifact?: unknown; - status: 'completed' | 'error'; + status: 'completed' | 'error' | 'cancelled'; }): Promise => { /** A provider id alone is not a durable part identity: it may * repeat in later turns of the same response. New automatic @@ -5610,7 +5625,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand return { taskId: task.id, toolName: tc.name, - status: params.status, + status: params.status === 'cancelled' ? 'error' : params.status, + ...(params.status === 'cancelled' ? { cancelled: true } : {}), settledAt: new Date(current?.updatedAt ?? Date.now()), ...(completionPreregistered ? { completionWakeup: true } : {}), ...(current?.resultClaim != null @@ -5743,7 +5759,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand * overwrite it. */ dispatchedAt: task.createdAt, codeExecutionContext, - ...(detachedReservation?.status === 'reserved' || !completionPreregistered + ...(detachedReservation?.status === 'reserved' || + (!completionPreregistered && params.status !== 'cancelled') ? {} : { backgroundTask, resolveBackgroundTask }), output: params.output ?? localTask?.result, @@ -5816,7 +5833,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand const persistSettledBackgroundResult = async (params: { output?: string; artifact?: unknown; - status: 'completed' | 'error'; + status: 'completed' | 'error' | 'cancelled'; }): Promise => { if (harvestEnabled) { await persistBackgroundResult(params); @@ -5838,7 +5855,6 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand } }; let invokePromise: Promise<{ content?: unknown; artifact?: unknown }>; - const backgroundAbortController = new AbortController(); try { invokePromise = Promise.resolve( tool.invoke(normalizedArgs, { @@ -5966,6 +5982,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand }; let producerRetirementTimeout: ReturnType | undefined; const requestBackgroundAbort = (): void => { + backgroundAbortSource ??= 'timeout'; backgroundAbortController.abort( new DOMException('Background task timed out', 'AbortError'), ); @@ -6103,12 +6120,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand const neutralizedError = filteredError?.errorMessage ?? errorOutput; const deliveredError = toBackgroundToolFailure(tc.name, neutralizedError); const registryError = isCodeCall ? deliveredError : neutralizedError; + /** Only an owner-authorized request is cancellation evidence. + * Providers and timeout controllers also use AbortError, so + * classifying by error shape would turn failures into a false + * claim that the owner cancelled the task. */ const detachedTerminalStatus = - toolError instanceof Error && - (toolError.name === 'AbortError' || - (toolError as Error & { code?: string }).code === 'ABORT_ERR') - ? 'cancelled' - : 'failed'; + backgroundAbortSource === 'manual' ? 'cancelled' : 'failed'; if ( !(await persistDetachedTerminal({ status: detachedTerminalStatus, @@ -6117,19 +6134,30 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand ) { return; } - backgroundTaskRegistry.fail( - backgroundUserId, - backgroundConversationId, - task.id, - registryError, - /** Failed code tasks join the heal path too: without this, - * a full-row save reverting the error patch would leave - * the dispatch card on the handle JSON forever. */ - { harvestStarted: harvestEnabled }, - ); + const settleOptions = { harvestStarted: harvestEnabled }; + if (detachedTerminalStatus === 'cancelled') { + backgroundTaskRegistry.cancel( + backgroundUserId, + backgroundConversationId, + task.id, + registryError, + settleOptions, + ); + } else { + backgroundTaskRegistry.fail( + backgroundUserId, + backgroundConversationId, + task.id, + registryError, + /** Failed code tasks join the heal path too: without this, + * a full-row save reverting the error patch would leave + * the dispatch card on the handle JSON forever. */ + settleOptions, + ); + } await persistSettledBackgroundResult({ output: deliveredError, - status: 'error', + status: detachedTerminalStatus === 'cancelled' ? 'cancelled' : 'error', }); await wakeDetachedActor(); } finally { @@ -6195,6 +6223,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand subagentTasks, claimBackgroundToolResult: backgroundToolCompletion?.claim, recoverDeadBackgroundToolClaim: backgroundToolCompletion?.recoverDeadClaim, + ordinaryToolCancellation, }); const taskSnapshot = getBackgroundTaskSnapshot({ userId: backgroundUserId, @@ -6362,12 +6391,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand } } if (persistBackgroundCodeResult && delivery.messageId) { - /** Error tasks carry their message in `error`, not + /** Error/cancelled tasks carry their message in `error`, not * `result`; abort-confirmed timeouts store it raw, so * wrap here — `toBackgroundToolFailure` is a no-op for * already-wrapped detached failures. */ const reapplyOutput = - delivery.status === 'error' + delivery.status === 'error' || delivery.status === 'cancelled' ? toBackgroundToolFailure( delivery.toolName, delivery.error ?? delivery.result ?? 'Background task failed', @@ -6382,6 +6411,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand agentId: delivery.agentId, output: reapplyOutput, attachments: delivery.attachments, + ...(delivery.backgroundTask == null + ? {} + : { backgroundTask: delivery.backgroundTask }), reapply: true, }).catch((reapplyError) => { logger.warn( diff --git a/packages/api/src/agents/harvest.ts b/packages/api/src/agents/harvest.ts index 0ebda3db75..55eb828bb2 100644 --- a/packages/api/src/agents/harvest.ts +++ b/packages/api/src/agents/harvest.ts @@ -33,7 +33,10 @@ export interface ProcessedCodeOutput { export interface BackgroundToolResultState { taskId: string; toolName: string; + /** Cancelled executions retain the pre-existing durable `error` state so + * older replicas can still claim them during a rolling deploy. */ status: 'completed' | 'error'; + cancelled?: true; settledAt: Date; /** This exact task owns a pre-registered automatic continuation delivery. */ completionWakeup?: true; diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts index 727526692e..5d3b1ae098 100644 --- a/packages/data-provider/src/config.spec.ts +++ b/packages/data-provider/src/config.spec.ts @@ -347,7 +347,10 @@ describe('agent background task config', () => { if (!result.success) { return; } - expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: true }); + expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ + completionWakeups: true, + ordinaryToolCancellation: false, + }); }); it('accepts an administrator poll-only policy', () => { @@ -360,7 +363,26 @@ describe('agent background task config', () => { if (!result.success) { return; } - expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: false }); + expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ + completionWakeups: false, + ordinaryToolCancellation: false, + }); + }); + + it('accepts administrator-enabled ordinary tool cancellation', () => { + const result = configSchema.safeParse({ + version: '1.0', + endpoints: { agents: { backgroundTasks: { ordinaryToolCancellation: true } } }, + }); + + expect(result.success).toBe(true); + if (!result.success) { + return; + } + expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ + completionWakeups: true, + ordinaryToolCancellation: true, + }); }); }); diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index e735aeaa76..d4f1c84fbe 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1336,6 +1336,9 @@ export const agentsEndpointSchema = baseEndpointSchema backgroundTasks: z .object({ completionWakeups: z.boolean().optional().default(true), + /** Cooperative cancellation for process-local ordinary tools. Off + * by default so existing deployments opt into the new control. */ + ordinaryToolCancellation: z.boolean().optional().default(false), }) .optional(), skills: z diff --git a/packages/data-provider/src/types/agents.ts b/packages/data-provider/src/types/agents.ts index e3b4fc8e23..f07e7944e2 100644 --- a/packages/data-provider/src/types/agents.ts +++ b/packages/data-provider/src/types/agents.ts @@ -93,6 +93,8 @@ export namespace Agents { taskId: string; toolName: string; status: 'completed' | 'error'; + /** Additive rolling-deploy-compatible cancellation discriminator. */ + cancelled?: true; settledAt: Date; resultClaim?: { kind: 'manual' | 'wakeup'; diff --git a/packages/data-schemas/src/methods/message.spec.ts b/packages/data-schemas/src/methods/message.spec.ts index c9c0d02f3b..033c94bbc3 100644 --- a/packages/data-schemas/src/methods/message.spec.ts +++ b/packages/data-schemas/src/methods/message.spec.ts @@ -1474,6 +1474,53 @@ describe('Message Operations', () => { expect(claim.status).toBe('acquired'); }); + it('claims a cancelled ordinary background-tool result as terminal evidence', async () => { + await saveMessage(mockCtx, { + ...mockMessageData, + content: [ + { + type: 'tool_call', + tool_call: { + id: 'call-cancelled-tool', + name: 'bash_tool', + output: 'Background task cancellation requested', + backgroundTask: { + version: 1, + taskId: 'task-cancelled-tool', + toolName: 'bash_tool', + status: 'error', + cancelled: true, + settledAt: new Date(), + completionWakeup: true, + }, + }, + }, + ], + }); + + await expect( + claimBackgroundToolResults({ + userId: 'user123', + conversationId: mockMessageData.conversationId as string, + messageId: 'msg123', + taskId: 'task-cancelled-tool', + kind: 'wakeup', + claimId: 'delivery-cancelled-tool', + }), + ).resolves.toEqual({ + status: 'acquired', + results: [ + { + taskId: 'task-cancelled-tool', + toolCallId: 'call-cancelled-tool', + toolName: 'bash_tool', + status: 'cancelled', + output: 'Background task cancellation requested', + }, + ], + }); + }); + it('elects one result consumer and batches terminal siblings for a wakeup', async () => { const terminal = (id: string, taskId: string, output: string) => ({ type: 'tool_call', diff --git a/packages/data-schemas/src/methods/message.ts b/packages/data-schemas/src/methods/message.ts index 48c9ee3166..ed0f3e0af9 100644 --- a/packages/data-schemas/src/methods/message.ts +++ b/packages/data-schemas/src/methods/message.ts @@ -459,7 +459,7 @@ export interface BackgroundToolResultRecord { taskId: string; toolCallId: string; toolName: string; - status: 'completed' | 'error'; + status: 'completed' | 'error' | 'cancelled'; output: string; agentId?: string; } @@ -647,6 +647,7 @@ export interface MessageMethods { taskId: string; toolName: string; status: 'completed' | 'error'; + cancelled?: true; settledAt: Date; completionWakeup?: true; resultClaim?: { @@ -1144,6 +1145,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa taskId: string; toolName: string; status: 'completed' | 'error'; + cancelled?: true; settledAt: Date; completionWakeup?: true; resultClaim?: { @@ -1193,6 +1195,8 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa partPatch['content.$[part].tool_call.backgroundTask.taskId'] = backgroundTask.taskId; partPatch['content.$[part].tool_call.backgroundTask.toolName'] = backgroundTask.toolName; partPatch['content.$[part].tool_call.backgroundTask.status'] = backgroundTask.status; + partPatch['content.$[part].tool_call.backgroundTask.cancelled'] = + backgroundTask.cancelled === true; partPatch['content.$[part].tool_call.backgroundTask.settledAt'] = backgroundTask.settledAt; if (backgroundTask.completionWakeup === true) { partPatch['content.$[part].tool_call.backgroundTask.completionWakeup'] = true; @@ -1401,6 +1405,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa taskId?: unknown; toolName?: unknown; status?: unknown; + cancelled?: unknown; resultClaim?: { kind?: unknown; claimId?: unknown }; }; }; @@ -1427,7 +1432,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa taskId: task.taskId, toolCallId: toolCall.id, toolName: task.toolName, - status: task.status, + status: task.cancelled === true ? 'cancelled' : task.status, output: typeof toolCall.output === 'string' ? toolCall.output : '', ...(resultAgentId == null ? {} : { agentId: resultAgentId }), });