🪓 feat: Cancel Background Tool Invocations (#15773)

* ✨ feat: Cancel ordinary background tools

* 🛡️ fix: Keep cancellation receipts rollout-safe

* 🛡️ fix: Harden Background Tool Cancellation

* 🧪 test: Cover cancellation config wiring

* 🎨 style: Sort cancellation imports

* 🧪 test: Align Cancellation Claim Result

* 🛑 fix: Complete Background Cancellation Lifecycle

* 🎨 style: Align Cancellation Test Formatting
This commit is contained in:
Danny Avila 2026-09-09 09:06:26 -04:00 • committed by GitHub
parent eaed216994
commit 559fd7d607
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 904 additions and 52 deletions

View file

@ -1281,6 +1281,7 @@ describe('OpenAIChatCompletionController', () => {
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
await OpenAIChatCompletionController(req, res);
const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1);
@ -1308,6 +1309,7 @@ describe('OpenAIChatCompletionController', () => {
);
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
await toolExecuteOptions.loadTools(['file_search'], 'agent-123');
expect(loadToolsForExecution).toHaveBeenLastCalledWith(
expect.objectContaining({

View file

@ -1872,6 +1872,7 @@ describe('createResponse controller', () => {
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
await createResponse(req, res);
const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1);
@ -1899,6 +1900,7 @@ describe('createResponse controller', () => {
);
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
await toolExecuteOptions.loadTools(['file_search'], 'agent-123');
expect(loadToolsForExecution).toHaveBeenLastCalledWith(
expect.objectContaining({

View file

@ -466,6 +466,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents];
const allowedProviders = new Set(agentsEConfig?.allowedProviders);
const ordinaryToolCancellationEnabled =
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
// Create tool loader
const loadTools = createToolLoader({ req, res, signal: execution.signal });
@ -801,6 +803,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
agent never gains sandbox access even if the admin enabled the
capability globally. */
const toolExecuteOptions = {
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,

View file

@ -703,6 +703,8 @@ const executeResponse = async (envelope, { req, res }) => {
codeWorkspaces: request.code_workspaces ?? req.resolvedConversation?.codeWorkspaces,
});
const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents];
const ordinaryToolCancellationEnabled =
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
const previousMessages = request.previous_response_id
? await loadPreviousMessages(request.previous_response_id, principal.userId)
: [];
@ -1144,6 +1146,7 @@ const executeResponse = async (envelope, { req, res }) => {
// Create tool execute options for event-driven tool execution
const toolExecuteOptions = {
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,
@ -1372,6 +1375,7 @@ const executeResponse = async (envelope, { req, res }) => {
});
const toolExecuteOptions = {
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,

View file

@ -183,6 +183,9 @@ const initializeClient = async ({
const completionWakeupsEnabled = backgroundCompletionWakeupsEnabled(
appConfig?.endpoints?.[EModelEndpoint.agents],
);
const ordinaryToolCancellationEnabled =
appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.ordinaryToolCancellation ===
true;
/** The normal controller resolves this once for timestamp anchoring. Reuse
* that trusted document for child-thread execution policy; resume and direct
* callers fall back to the same owner-scoped lookup. */
@ -407,6 +410,7 @@ const initializeClient = async ({
// is composed with this authoritative job signal by the handler.
runSignal: signal,
foregroundRunId: runtimeRequestBody?.messageId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
loadTools: async (toolNames, agentId, _configurable, callerCapabilityProjection) => {
const ctx = agentToolContexts.get(agentId) ?? {};
logger.debug(`[ON_TOOL_EXECUTE] ctx found: ${!!ctx.userMCPAuthMap}, agent: ${ctx.agent?.id}`);

View file

@ -321,6 +321,31 @@ describe('initializeClient — processAgent ACL gate', () => {
expect(capturedToolExecuteOptions.foregroundRunId).toBe('response-1');
});
it('threads the deployment ordinary-tool cancellation gate into execution', async () => {
mockInitializeAgent.mockResolvedValue(makePrimaryConfig([]));
const disabledReq = makeReq();
await initializeClient({
req: disabledReq,
res: {},
signal: new AbortController().signal,
endpointOption: makeEndpointOption(),
});
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(false);
const enabledReq = makeReq();
enabledReq.config.endpoints.agents = {
backgroundTasks: { ordinaryToolCancellation: true },
};
await initializeClient({
req: enabledReq,
res: {},
signal: new AbortController().signal,
endpointOption: makeEndpointOption(),
});
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(true);
});
it('propagates an expected-MCP-tools failure from the runtime tool loader', async () => {
const toolError = Object.assign(new Error('Expected MCP tools are unavailable'), {
code: 'AGENT_EXPECTED_MCP_TOOLS_UNAVAILABLE',

View file

@ -209,6 +209,7 @@ const Part = memo(function Part({
runStepStatus={toolCall.runStepStatus}
runStepDurationMs={toolCall.runStepDurationMs}
backgrounded={toolCall.backgrounded}
backgroundCancelled={toolCall.backgroundTask?.cancelled === true}
attachments={attachments}
commandField="code"
hideAttachments={hideAttachments}
@ -228,6 +229,7 @@ const Part = memo(function Part({
runStepStatus={toolCall.runStepStatus}
runStepDurationMs={toolCall.runStepDurationMs}
backgrounded={toolCall.backgrounded}
backgroundCancelled={toolCall.backgroundTask?.cancelled === true}
output={toolCall.output ?? ''}
initialProgress={toolCall.progress ?? 0.1}
args={toolCall.args}
@ -361,6 +363,7 @@ const Part = memo(function Part({
runStepStatus={toolCall.runStepStatus}
runStepDurationMs={toolCall.runStepDurationMs}
backgrounded={toolCall.backgrounded}
backgroundCancelled={toolCall.backgroundTask?.cancelled === true}
attachments={attachments}
hideAttachments={hideAttachments}
onExpand={onToolExpand}
@ -410,7 +413,9 @@ const Part = memo(function Part({
isLast={isLast}
hideAttachments={hideAttachments}
onExpand={onToolExpand}
runStepStatus={toolCall.runStepStatus}
runStepStatus={
toolCall.backgroundTask?.cancelled === true ? 'cancelled' : toolCall.runStepStatus
}
runStepDurationMs={toolCall.runStepDurationMs}
/>
);

View file

@ -25,6 +25,7 @@ export default function BashCall({
runStepStatus,
runStepDurationMs,
backgrounded,
backgroundCancelled = false,
initialProgress = 0.1,
args,
output = '',
@ -39,6 +40,7 @@ export default function BashCall({
runStepStatus?: PartMetadata['runStepStatus'];
runStepDurationMs?: PartMetadata['runStepDurationMs'];
backgrounded?: PartMetadata['backgrounded'];
backgroundCancelled?: boolean;
args?: string | Record<string, unknown>;
output?: string;
attachments?: TAttachment[];
@ -67,6 +69,8 @@ export default function BashCall({
[attachments, toolCallId],
);
const backgroundFailed = backgroundHandle != null && backgroundStatus === 'error';
const cancelledInBackground =
backgroundCancelled || (backgroundHandle != null && backgroundStatus === 'cancelled');
const backgroundFinishedText = backgroundHandle
? localize(
backgroundStatus != null || (fileAttachments?.length ?? 0) > 0
@ -83,6 +87,7 @@ export default function BashCall({
onExpand,
runStepStatus,
extraError: backgroundFailed,
extraCancelled: cancelledInBackground,
});
const highlighted = useLazyHighlight(command || undefined, 'bash');

View file

@ -63,6 +63,7 @@ export default function ExecuteCode({
runStepStatus,
runStepDurationMs,
backgrounded,
backgroundCancelled = false,
initialProgress = 0.1,
args,
output = '',
@ -76,6 +77,7 @@ export default function ExecuteCode({
runStepStatus?: PartMetadata['runStepStatus'];
runStepDurationMs?: PartMetadata['runStepDurationMs'];
backgrounded?: PartMetadata['backgrounded'];
backgroundCancelled?: boolean;
args?: string | Record<string, unknown>;
output?: string;
attachments?: TAttachment[];
@ -105,6 +107,8 @@ export default function ExecuteCode({
[attachments, toolCallId],
);
const backgroundFailed = backgroundHandle != null && backgroundStatus === 'error';
const cancelledInBackground =
backgroundCancelled || (backgroundHandle != null && backgroundStatus === 'cancelled');
const backgroundFinishedText = backgroundHandle
? localize(
backgroundStatus != null || (fileAttachments?.length ?? 0) > 0
@ -121,6 +125,7 @@ export default function ExecuteCode({
onExpand,
runStepStatus,
extraError: backgroundFailed,
extraCancelled: cancelledInBackground,
});
const highlighted = useLazyHighlight(code, lang);

View file

@ -261,6 +261,36 @@ describe('BashCall backgrounded calls', () => {
expect(screen.getByTestId('progress-text')).toHaveTextContent('tool failed');
});
it('surfaces a live cancelled marker as cancellation rather than failure', () => {
renderBackgrounded([
{
type: 'background_task_status',
file_id: 'bg-tc-1',
toolCallId: 'tc-1',
status: 'cancelled',
},
]);
expect(screen.getByTestId('progress-text')).toHaveTextContent('Cancelled');
expect(screen.getByTestId('progress-text')).not.toHaveTextContent('tool failed');
});
it('restores a persisted background cancellation as cancellation', () => {
render(
<RecoilRoot>
<BashCall
initialProgress={1}
isSubmitting={false}
args={{ command: 'sleep 600' }}
output="Error: [bash_tool] tool call failed: Background task cancellation requested"
backgrounded={true}
backgroundCancelled={true}
/>
</RecoilRoot>,
);
expect(screen.getByTestId('progress-text')).toHaveTextContent('Cancelled');
expect(screen.getByTestId('progress-text')).not.toHaveTextContent('tool failed');
});
it('renders real stdout normally after the background result patches the output', () => {
render(
<RecoilRoot>

View file

@ -75,6 +75,16 @@ describe('parseBackgroundHandle', () => {
expect(fileAttachments).toHaveLength(0);
});
it('preserves the cancelled terminal status from an owned marker', () => {
const marker = {
type: 'background_task_status',
file_id: 'bg-tc-mine',
toolCallId: 'tc-mine',
status: 'cancelled',
} as never;
expect(splitBackgroundAttachments([marker], 'tc-mine').backgroundStatus).toBe('cancelled');
});
it('returns null for oversized payloads (real output, not a handle)', () => {
const big = JSON.stringify({
background_task_id: 'task-1',

View file

@ -37,6 +37,10 @@ export interface UseToolCallStateInput {
* every other card instead of patching the result afterwards.
*/
extraError?: boolean;
/** A terminal verdict carried outside the dispatch run step, such as a
* cancelled ordinary background tool. Cancellation outranks error-shaped
* output so the card never relabels an intentional stop as failure. */
extraCancelled?: boolean;
}
export default function useToolCallState({
@ -47,6 +51,7 @@ export default function useToolCallState({
onExpand,
runStepStatus,
extraError = false,
extraCancelled = false,
}: UseToolCallStateInput): ToolCallState {
const autoExpand = useRecoilValue(store.autoExpandTools);
const hasOutput = output.length > 0;
@ -86,7 +91,7 @@ export default function useToolCallState({
* for why the cancellation inference must not read the animated one.
*/
const phase = resolveToolCallPhase({
runStepStatus,
runStepStatus: extraCancelled ? 'cancelled' : runStepStatus,
displayProgress: rawProgress,
reportedProgress: initialProgress,
isSubmitting,

View file

@ -148,20 +148,23 @@ function getToolMeta(
* reported no failed action beside a card showing failure. Correlated the
* same way the child is, since provider tool-call ids repeat across agents
* and execution steps in handoff responses. */
const backgroundFailed =
parseBackgroundHandle(tc.output) != null &&
splitBackgroundAttachments(
filterAttachmentsForPart(
attachmentsByToolCallId?.[tc.id ?? ''],
tc.agentId,
toolCall.stepId,
),
tc.id,
).backgroundStatus === 'error';
const backgroundHandle = parseBackgroundHandle(tc.output);
const backgroundStatus = splitBackgroundAttachments(
filterAttachmentsForPart(attachmentsByToolCallId?.[tc.id ?? ''], tc.agentId, toolCall.stepId),
tc.id,
).backgroundStatus;
const backgroundFailed = backgroundHandle != null && backgroundStatus === 'error';
const backgroundCancelled =
tc.backgroundTask?.cancelled === true ||
(backgroundHandle != null && backgroundStatus === 'cancelled');
return {
name,
iconName,
...resolveOutcome(runStepStatus, completed, failedOutput || backgroundFailed),
...resolveOutcome(
backgroundCancelled ? 'cancelled' : runStepStatus,
completed,
failedOutput || backgroundFailed,
),
};
}

View file

@ -57,7 +57,9 @@ jest.mock('../WebSearch', () => ({
jest.mock('../ToolCall', () => ({
__esModule: true,
default: () => <div data-testid="tool-call" />,
default: ({ runStepStatus }: { runStepStatus?: string }) => (
<div data-testid="tool-call" data-run-step-status={runStepStatus} />
),
}));
jest.mock('../Image', () => ({
@ -135,6 +137,21 @@ describe('Part tool renderer selection', () => {
expect(screen.queryByTestId('tool-call')).not.toBeInTheDocument();
});
it('renders a cancelled generic background tool as cancelled', () => {
const part = toolCallPart('search_mcp_docs') as Extract<
TMessageContentParts,
{ type: typeof ContentTypes.TOOL_CALL }
>;
Object.assign(part[ContentTypes.TOOL_CALL], {
runStepStatus: 'failed',
backgroundTask: { cancelled: true },
});
renderPart(part);
expect(screen.getByTestId('tool-call')).toHaveAttribute('data-run-step-status', 'cancelled');
});
it('routes an unavailable reasoning marker to the marker renderer', () => {
renderPart({
type: ContentTypes.THINK,

View file

@ -961,6 +961,39 @@ describe('ToolCallGroup image hoisting', () => {
).toBeInTheDocument();
});
it('counts a persisted ordinary background cancellation as cancelled', () => {
const cancelled = {
type: ContentTypes.TOOL_CALL,
[ContentTypes.TOOL_CALL]: {
id: 'c2',
name: 'bash_tool',
args: '{"command":"sleep 600"}',
output: 'Error: [bash_tool] tool call failed: Background task cancellation requested',
runStepStatus: 'completed',
backgrounded: true,
backgroundTask: {
version: 1,
taskId: 'background-task-1',
toolName: 'bash_tool',
status: 'error',
cancelled: true,
settledAt: new Date(),
},
},
} as unknown as TMessageContentParts;
renderGroup({
...baseProps,
parts: [
{ part: makePart('c1', 'created', 'create_file'), idx: 0 },
{ part: cancelled, idx: 1 },
],
lastContentIdx: 1,
});
expect(screen.getByRole('button', { name: /· 1 cancelled$/ })).toBeInTheDocument();
});
it('labels a homogeneous ask_user_question group as its own category', () => {
renderGroup({
...baseProps,

View file

@ -652,6 +652,8 @@ endpoints:
# # Enabled by default; set false to keep background tools and subagents poll-only.
# backgroundTasks:
# completionWakeups: true
# # Allow owners to cooperatively cancel ordinary background tools, including BYOM Bash.
# ordinaryToolCancellation: false
# # "run_in_background" makes Code Interpreter tools eligible by default and enables per-tool MCP opt-in.
# # "tool_intents" enables live model-written labels for native tools and opted-in MCP tools.
# # (optional) Require user approval before matching tool calls. Disabled by default.

View file

@ -675,6 +675,120 @@ describe('selection policy at injection time', () => {
});
describe('BackgroundTaskRegistryClass', () => {
it('requests cancellation idempotently without settling or releasing running capacity', () => {
const registry = new BackgroundTaskRegistryClass();
const requestCancellation = jest.fn();
const created = registry.create({
userId: 'cancel-owner',
conversationId: 'cancel-conversation',
toolCallId: 'cancel-call',
toolName: 'bash_tool',
requestCancellation,
});
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
expect(
registry.requestCancellation('other-owner', 'cancel-conversation', created.task.id),
).toEqual({ status: 'not_found' });
expect(
registry.requestCancellation('cancel-owner', 'other-conversation', created.task.id),
).toEqual({ status: 'not_found' });
expect(
registry.requestCancellation('cancel-owner', 'cancel-conversation', created.task.id).status,
).toBe('requested');
expect(
registry.requestCancellation('cancel-owner', 'cancel-conversation', created.task.id).status,
).toBe('already_requested');
expect(requestCancellation).toHaveBeenCalledTimes(1);
expect(created.task).toMatchObject({
status: 'running',
cancellationRequestedAt: expect.any(Number),
});
for (let index = 0; index < 9; index += 1) {
const admitted = registry.create({
userId: 'cancel-owner',
conversationId: 'cancel-conversation',
toolCallId: `other-call-${index}`,
toolName: 'background-tool',
});
expect('atCapacity' in admitted).toBe(false);
}
expect(
registry.create({
userId: 'cancel-owner',
conversationId: 'cancel-conversation',
toolCallId: 'blocked-until-settlement',
toolName: 'background-tool',
}),
).toEqual({ atCapacity: true, scope: 'conversation_running' });
registry.cancel(
'cancel-owner',
'cancel-conversation',
created.task.id,
'Background task cancellation requested',
);
expect(created.task.status).toBe('cancelled');
expect(
registry.create({
userId: 'cancel-owner',
conversationId: 'cancel-conversation',
toolCallId: 'admitted-after-settlement',
toolName: 'background-tool',
}),
).toMatchObject({ isNew: true });
expect(
registry.requestCancellation('cancel-owner', 'cancel-conversation', created.task.id).status,
).toBe('settled');
});
it('lets actual completion win when an abort-resistant invocation settles successfully', () => {
const registry = new BackgroundTaskRegistryClass();
const created = registry.create({
userId: 'race-owner',
conversationId: 'race-conversation',
toolCallId: 'race-call',
toolName: 'mutation',
requestCancellation: jest.fn(),
});
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
registry.requestCancellation('race-owner', 'race-conversation', created.task.id);
registry.complete('race-owner', 'race-conversation', created.task.id, {
content: 'completed despite cancellation request',
});
registry.cancel('race-owner', 'race-conversation', created.task.id, 'cancelled too late');
expect(created.task).toMatchObject({
status: 'completed',
result: 'completed despite cancellation request',
});
});
it('does not record a manual cancellation when another abort source already won', () => {
const registry = new BackgroundTaskRegistryClass();
const created = registry.create({
userId: 'timeout-owner',
conversationId: 'timeout-conversation',
toolCallId: 'timeout-call',
toolName: 'mutation',
requestCancellation: () => false,
});
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
expect(
registry.requestCancellation('timeout-owner', 'timeout-conversation', created.task.id),
).toMatchObject({ status: 'unavailable' });
expect(created.task.cancellationRequestedAt).toBeUndefined();
});
it('creates, completes, and reads a task', () => {
const registry = new BackgroundTaskRegistryClass();
const created = registry.create({
@ -1970,6 +2084,10 @@ describe('getBackgroundCodeDelivery (singleton)', () => {
messageId: 'dispatch-msg',
result: 'stdout',
attachments: [{ file_id: 'f1' }],
backgroundTask: expect.objectContaining({
taskId: created.task.id,
status: 'completed',
}),
}),
);
/** Not one-shot: a later poll can still re-emit / re-anchor. */
@ -2017,6 +2135,84 @@ describe('runCheckBackgroundTask (singleton)', () => {
);
});
it('keeps ordinary cancellation disabled unless the deployment opts in', async () => {
const requestCancellation = jest.fn();
const created = backgroundTaskRegistry.create({
userId: 'disabled-cancel-user',
conversationId: 'disabled-cancel-conversation',
toolCallId: 'disabled-cancel-call',
toolName: 'bash_tool',
requestCancellation,
});
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
const result = JSON.parse(
await runCheckBackgroundTask({
userId: 'disabled-cancel-user',
conversationId: 'disabled-cancel-conversation',
args: { background_task_id: created.task.id, action: 'cancel' },
}),
);
expect(result).toMatchObject({ status: 'invalid', background_task_id: created.task.id });
expect(requestCancellation).not.toHaveBeenCalled();
backgroundTaskRegistry.fail(
'disabled-cancel-user',
'disabled-cancel-conversation',
created.task.id,
'test cleanup',
);
});
it('uses normal durable claim arbitration when cancellation loses the settlement race', async () => {
const created = backgroundTaskRegistry.create({
userId: 'cancel-race-user',
conversationId: 'cancel-race-conversation',
toolCallId: 'cancel-race-call',
toolName: 'bash_tool',
messageId: 'cancel-race-message',
requestCancellation: jest.fn(),
});
if ('atCapacity' in created) {
throw new Error('unexpected capacity');
}
backgroundTaskRegistry.markCompletionWakeup(
'cancel-race-user',
'cancel-race-conversation',
created.task.id,
{
renew: jest.fn(async () => true),
retire: jest.fn(async () => true),
},
);
backgroundTaskRegistry.complete(
'cancel-race-user',
'cancel-race-conversation',
created.task.id,
{ content: 'settled result' },
);
const claimBackgroundToolResult = jest.fn(async () => ({
status: 'claimed' as const,
claim: { kind: 'wakeup' as const, claimId: 'automatic-delivery' },
}));
const result = JSON.parse(
await runCheckBackgroundTask({
userId: 'cancel-race-user',
conversationId: 'cancel-race-conversation',
args: { background_task_id: created.task.id, action: 'cancel' },
ordinaryToolCancellation: true,
claimBackgroundToolResult,
}),
);
expect(result).toMatchObject({
status: 'delivery_scheduled',
background_task_id: created.task.id,
});
expect(claimBackgroundToolResult).toHaveBeenCalledTimes(1);
});
it('rejects an oversized task id before local or cross-replica lookup', async () => {
const store = Object.assign(new InMemorySubagentTaskStore(), {
claimTask: jest.fn(),

View file

@ -53,6 +53,7 @@ import type {
BackgroundToolResultRecord,
} from '@librechat/data-schemas';
import type { AgentToolOptions } from 'librechat-data-provider';
import type { BackgroundToolResultState } from './harvest';
import type { CapabilityToolNames } from './selection';
import {
BACKGROUND_TASK_TIMEOUT_MS,
@ -353,7 +354,7 @@ export function stripBackgroundFromToolRegistry(
const CHECK_BACKGROUND_TASK_DESCRIPTION = `Check, control, and retrieve tool or subagent tasks previously dispatched in the background (with run_in_background: true).
Provide a background_task_id to poll one task; omit it to list every background task in this thread. A task is only finished when its status is "completed", "error", or "cancelled" — never assume completion without polling. Results are not pushed to you; you must call this tool to collect them. Subagent tasks additionally accept steer, queue, interrupt, cancel, and cancel_message actions while running. Live subagent controls route across API replicas but do not survive a restart of the process that owns the executor. A completed subagent thread may be continued later through the subagent tool's durable thread id.`;
Provide a background_task_id to poll one task; omit it to list every background task in this thread. A task is only finished when its status is "completed", "error", or "cancelled" — never assume completion without polling. Results are not pushed to you; you must call this tool to collect them. The cancel action applies to subagents and to ordinary tools when enabled by the deployment; steer, queue, interrupt, and cancel_message apply only to subagents. Live controls route to process-local executors and do not survive an owning-process restart. A completed subagent thread may be continued later through the subagent tool's durable thread id.`;
const CHECK_BACKGROUND_TASK_WAKEUP_DESCRIPTION = `Check, control, and retrieve tool or subagent tasks previously dispatched in the background (with run_in_background: true).
@ -399,7 +400,8 @@ const CHECK_BACKGROUND_TASK_PARAMETERS = Object.freeze<CheckBackgroundTaskParame
action: {
type: 'string',
enum: ['poll', 'steer', 'queue', 'interrupt', 'cancel', 'cancel_message'],
description: 'Defaults to poll. Control actions apply only to a running subagent task.',
description:
'Defaults to poll. Cancel applies to subagents and may apply to ordinary tools when enabled by the deployment; other controls apply only to subagents.',
},
message: {
type: 'string',
@ -600,7 +602,7 @@ export function synthesizeBackgroundToolOptions(sources: {
);
}
export type BackgroundTaskStatus = 'running' | 'completed' | 'error';
export type BackgroundTaskStatus = 'running' | 'completed' | 'error' | 'cancelled';
export interface BackgroundTask {
id: string;
@ -614,6 +616,10 @@ export interface BackgroundTask {
* (e.g. `call_0`) across agents when patching the dispatch turn. */
agentId?: string;
status: BackgroundTaskStatus;
/** Set when an owner asked the live invocation to abort. The task remains
* running until the invocation actually settles, so capacity and status do
* not imply that external effects have stopped prematurely. */
cancellationRequestedAt?: number;
/** Tool result content once completed. */
result?: string;
/**
@ -817,6 +823,9 @@ function toStoredArtifact(
export class BackgroundTaskRegistryClass {
private readonly buckets = new Map<string, TaskBucket>();
private readonly retainedUsage = new WeakMap<BackgroundTask, RetainedPayloadUsage>();
/** Live invocation controls are intentionally process-local and are never
* exposed through task snapshots or durable receipts. */
private readonly cancellationRequests = new WeakMap<BackgroundTask, () => boolean | void>();
private lastGlobalSweepAt = 0;
private key(userId: string, conversationId: string): string {
@ -1213,6 +1222,10 @@ export class BackgroundTaskRegistryClass {
harvestStarted?: boolean;
liveArtifactPollRequired?: boolean;
capacityPermit?: BackgroundTaskCapacityPermit;
/** Abort the already-created invocation signal. Ownership is enforced by
* requestCancellation before this closure can be reached. */
/** Returns false when another abort source already won the race. */
requestCancellation?: () => boolean | void;
}): { task: BackgroundTask; isNew: boolean } | BackgroundTaskCapacityRejection {
const now = Date.now();
this.sweep(now);
@ -1286,9 +1299,42 @@ export class BackgroundTaskRegistryClass {
};
bucket.tasks.set(task.id, task);
bucket.byToolCall.set(dedupeKey, task.id);
if (params.requestCancellation != null) {
this.cancellationRequests.set(task, params.requestCancellation);
}
return { task, isNew: true };
}
requestCancellation(
userId: string,
conversationId: string,
taskId: string,
):
| { status: 'not_found' }
| { status: 'unavailable'; task: BackgroundTask }
| { status: 'requested' | 'already_requested' | 'settled'; task: BackgroundTask } {
const task = this.get(userId, conversationId, taskId);
if (task == null) {
return { status: 'not_found' };
}
if (task.status !== 'running') {
return { status: 'settled', task };
}
if (task.cancellationRequestedAt != null) {
return { status: 'already_requested', task };
}
const request = this.cancellationRequests.get(task);
if (request == null) {
return { status: 'unavailable', task };
}
if (request() === false) {
return { status: 'unavailable', task };
}
task.cancellationRequestedAt = Date.now();
task.updatedAt = task.cancellationRequestedAt;
return { status: 'requested', task };
}
private update(
userId: string,
conversationId: string,
@ -1338,6 +1384,7 @@ export class BackgroundTaskRegistryClass {
artifactDelivered: false,
});
if (updated) {
this.cancellationRequests.delete(task);
this.updatePayloadUsage(task, {
result: retainedContent?.length ?? 0,
artifact: artifactChars,
@ -1508,6 +1555,40 @@ export class BackgroundTaskRegistryClass {
...(options?.harvestStarted === true ? { harvestStarted: true, harvestPending: true } : {}),
});
if (updated) {
this.cancellationRequests.delete(task);
this.retainedUsage.set(task, {
result: 0,
artifact: 0,
attachments: 0,
error: retainedError?.length ?? 0,
});
}
}
cancel(
userId: string,
conversationId: string,
taskId: string,
error: string,
options?: { harvestStarted?: boolean },
): void {
const storedError = truncateMiddle(error, MAX_RESULT_CHARS);
const task = this.buckets.get(this.key(userId, conversationId))?.tasks.get(taskId);
if (task == null || task.status !== 'running' || task.artifactBlocked === true) {
return;
}
const hasRetainedCapacity = this.makeRetainedRoom(userId, task, storedError.length);
const retainedError = hasRetainedCapacity ? storedError : undefined;
const updated = this.update(userId, conversationId, taskId, {
status: 'cancelled',
error: retainedError,
result: undefined,
artifact: undefined,
attachments: undefined,
...(options?.harvestStarted === true ? { harvestStarted: true, harvestPending: true } : {}),
});
if (updated) {
this.cancellationRequests.delete(task);
this.retainedUsage.set(task, {
result: 0,
artifact: 0,
@ -1749,6 +1830,7 @@ interface SerializedBackgroundTask {
status: BackgroundTaskStatus;
/** Coarse 0..1: no intermediate progress exists, only running vs settled. */
progress: number;
cancellation_requested?: boolean;
result?: string;
result_available?: boolean;
result_chars?: number;
@ -1795,6 +1877,9 @@ function serializeTask(
tool: task.toolName,
status: task.status,
progress: task.status === 'running' ? 0 : 1,
...(task.status === 'running' && task.cancellationRequestedAt != null
? { cancellation_requested: true }
: {}),
...resultFields(task, includeResult),
...taskNote(task),
...(task.error !== undefined ? { error: task.error } : {}),
@ -1982,6 +2067,8 @@ export async function runCheckBackgroundTask(params: {
allowUnfinished?: boolean;
}) => Promise<BackgroundToolResultClaim>;
recoverDeadBackgroundToolClaim?: BackgroundToolDeadClaimRecovery;
/** Trusted deployment policy. Defaults false for backward compatibility. */
ordinaryToolCancellation?: boolean;
}): Promise<string> {
const { userId, conversationId } = params;
const args = coerceArgsObject(params.args) ?? {};
@ -2000,11 +2087,48 @@ export async function runCheckBackgroundTask(params: {
const task = backgroundTaskRegistry.get(userId, conversationId, taskId);
if (task != null) {
if (action !== 'poll') {
return JSON.stringify({
status: 'invalid',
background_task_id: taskId,
message: 'Control actions are supported only for subagent tasks.',
});
if (action === 'cancel') {
if (params.ordinaryToolCancellation !== true) {
return JSON.stringify({
status: 'invalid',
background_task_id: taskId,
message: 'Cancellation is not enabled for ordinary background tools.',
});
}
const cancellation = backgroundTaskRegistry.requestCancellation(
userId,
conversationId,
taskId,
);
if (cancellation.status === 'settled') {
/** Settlement won the race. Fall through to the ordinary poll path
* so durable result-claim arbitration still elects one consumer. */
} else if (
cancellation.status === 'requested' ||
cancellation.status === 'already_requested'
) {
return JSON.stringify({
...serializeTask(cancellation.task, { includeResult: false }),
status: 'cancellation_requested',
cancellation_requested: true,
message:
'Cancellation was requested. The task remains active until its executor settles; poll again for a terminal result.',
});
} else {
return JSON.stringify({
status: 'unavailable',
background_task_id: taskId,
message:
'This server no longer owns a live cancellation handle for the task. Poll for its outcome; do not assume execution stopped.',
});
}
} else {
return JSON.stringify({
status: 'invalid',
background_task_id: taskId,
message: 'This control action is supported only for subagent tasks.',
});
}
}
if (task.status !== 'running') {
if (
@ -2545,6 +2669,7 @@ export function getBackgroundCodeDelivery(params: {
result?: string;
error?: string;
attachments?: unknown[];
backgroundTask?: BackgroundToolResultState;
}
| undefined {
const rawId = coerceArgsObject(params.args)?.background_task_id;
@ -2572,6 +2697,30 @@ export function getBackgroundCodeDelivery(params: {
result: task.result,
error: task.error,
attachments: task.attachments,
...(task.status === 'running'
? {}
: {
backgroundTask: {
taskId: task.id,
toolName: task.toolName,
status: task.status === 'cancelled' ? 'error' : task.status,
...(task.status === 'cancelled' ? { cancelled: true } : {}),
settledAt: new Date(task.updatedAt),
...(task.completionWakeup === true ? { completionWakeup: true } : {}),
...(task.resultClaim == null
? {}
: {
resultClaim: {
kind: task.resultClaim.kind,
claimId: task.resultClaim.claimId,
claimedAt: new Date(task.resultClaim.claimedAt),
...(task.resultClaim.generationId == null
? {}
: { generationId: task.resultClaim.generationId }),
},
}),
},
}),
};
}

View file

@ -218,7 +218,7 @@ function buildWakeupInput(
taskId: string;
toolCallId: string;
toolName: string;
status: 'completed' | 'error';
status: 'completed' | 'error' | 'cancelled';
output: string;
}>,
): string {

View file

@ -2,6 +2,7 @@ import { z } from 'zod';
import { logger } from '@librechat/data-schemas';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
import type { ToolExecuteOptions } from './handlers';
import {
backgroundTaskRegistry,
runCheckBackgroundTask,
@ -618,6 +619,95 @@ describe('createToolExecuteHandler — background tool calls', () => {
}
});
it('preserves timeout provenance when cancellation is requested after the deadline', async () => {
jest.useFakeTimers({ doNotFake: ['setImmediate'] });
try {
let rejectInvocation: ((error: unknown) => void) | undefined;
let invocationSignal: AbortSignal | undefined;
const tool = {
name: 'execute_code',
description: 'settles after acknowledging an abort',
schema: z.object({ lang: z.string(), code: z.string() }),
invoke: jest.fn(
(_input: unknown, config?: { signal?: AbortSignal }) =>
new Promise((_resolve, reject) => {
invocationSignal = config?.signal;
rejectInvocation = reject;
}),
),
} as unknown as StructuredToolInterface;
const persistBackgroundCodeResult = jest.fn(
async (
_params: Parameters<NonNullable<ToolExecuteOptions['persistBackgroundCodeResult']>>[0],
) => ({ attachments: [] }),
);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
ordinaryToolCancellation: true,
persistBackgroundCodeResult,
backgroundToolCompletion: {
preregister: async () => ({
renew: jest.fn(async () => true),
retire: jest.fn(async () => true),
}),
persist: jest.fn(async () => true),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
const configurable = buildConfig([tool.name]);
const metadata = { thread_id: 'late_cancel_conversation', run_id: 'late-cancel-run' };
const [dispatch] = await runBatch(handler, {
toolCalls: [
{
id: 'call_late_cancel',
name: 'execute_code',
args: { lang: 'python', code: 'while True: pass', run_in_background: true },
},
],
agentId: 'agent_late_cancel',
configurable,
metadata,
});
const taskId = JSON.parse(dispatch.content).background_task_id as string;
jest.advanceTimersByTime(31 * 60 * 1000);
await flushMicrotasks();
const [cancellation] = await runBatch(handler, {
toolCalls: [
{
id: 'call_late_cancel_control',
name: CHECK_BACKGROUND_TASK_NAME,
args: { background_task_id: taskId, action: 'cancel' },
},
],
agentId: 'agent_late_cancel',
configurable,
metadata,
});
expect(JSON.parse(cancellation.content)).toMatchObject({
status: 'unavailable',
background_task_id: taskId,
});
rejectInvocation?.(
invocationSignal?.reason ?? new DOMException('Background task timed out', 'AbortError'),
);
await flushMicrotasks();
await flushMicrotasks();
const persisted =
persistBackgroundCodeResult.mock.calls[
persistBackgroundCodeResult.mock.calls.length - 1
]?.[0];
expect(persisted).toEqual(
expect.objectContaining({ output: expect.stringContaining('Background task timed out') }),
);
expect(persisted?.backgroundTask?.cancelled).not.toBe(true);
} finally {
jest.useRealTimers();
}
});
it('retires the automatic wakeup after an abort-resistant invocation exceeds its grace period', async () => {
jest.useFakeTimers({ doNotFake: ['setImmediate'] });
try {
@ -977,7 +1067,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(eventActorDetachedAction.wake).toHaveBeenCalledTimes(1);
});
it('records an aborted detached expected action as cancelled', async () => {
it('does not claim an unrequested provider abort was owner cancellation', async () => {
const abortError = Object.assign(new Error('operation aborted'), { name: 'AbortError' });
const tool = {
name: 'submit_move_mcp_chess',
@ -1018,7 +1108,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
await flushMicrotasks();
expect(eventActorDetachedAction.settle).toHaveBeenCalledWith(
expect.objectContaining({ status: 'cancelled', error: 'operation aborted' }),
expect.objectContaining({ status: 'failed', error: 'operation aborted' }),
);
});
@ -1074,6 +1164,154 @@ describe('createToolExecuteHandler — background tool calls', () => {
expect(polled.result).toContain('RESULT for librechat');
});
it('cancels a running background Bash mutation, reports settlement truthfully, and reuses the workspace', async () => {
let delayedWrites = 0;
let invocations = 0;
const bashTool = {
name: 'bash_tool',
description: 'runs a workspace command',
schema: z.object({ command: z.string() }),
invoke: jest.fn(
async (
input: { command: string },
config?: { signal?: AbortSignal },
): Promise<{ content: string }> => {
invocations += 1;
if (input.command === 'pwd') {
return { content: '/workspace' };
}
return await new Promise((resolve, reject) => {
const timer = setTimeout(() => {
delayedWrites += 1;
resolve({ content: 'late mutation completed' });
}, 50);
config?.signal?.addEventListener(
'abort',
() => {
clearTimeout(timer);
reject(
config.signal?.reason ??
new DOMException('Background task cancellation requested', 'AbortError'),
);
},
{ once: true },
);
});
},
),
} as unknown as StructuredToolInterface;
const persistBackgroundCodeResult = jest.fn(async () => ({ attachments: [] }));
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [bashTool] }),
ordinaryToolCancellation: true,
persistBackgroundCodeResult,
});
const configurable = buildConfig(['bash_tool']);
const metadata = { thread_id: 'cancel_bash_conversation', run_id: 'cancel-bash-run' };
const dispatch = await runBatch(handler, {
toolCalls: [
{
id: 'call_cancel_bash',
name: 'bash_tool',
stepId: 'step_cancel_bash',
args: { command: 'sleep-then-write', run_in_background: true },
},
],
agentId: 'agent_cancel_bash',
configurable,
metadata,
});
const taskId = JSON.parse(dispatch[0].content).background_task_id as string;
const cancellation = JSON.parse(
(
await runBatch(handler, {
toolCalls: [
{
id: 'call_cancel_control',
name: CHECK_BACKGROUND_TASK_NAME,
args: { background_task_id: taskId, action: 'cancel' },
},
],
agentId: 'agent_cancel_bash',
configurable,
metadata,
})
)[0].content,
);
expect(cancellation).toMatchObject({
status: 'cancellation_requested',
cancellation_requested: true,
});
await flushMicrotasks();
await flushMicrotasks();
await new Promise((resolve) => setTimeout(resolve, 75));
const terminal = JSON.parse(
(
await runBatch(handler, {
toolCalls: [
{
id: 'call_poll_cancelled_bash',
name: CHECK_BACKGROUND_TASK_NAME,
args: { background_task_id: taskId },
},
],
agentId: 'agent_cancel_bash',
configurable,
metadata,
})
)[0].content,
);
expect(terminal).toMatchObject({
status: 'cancelled',
background_task_id: taskId,
});
expect(persistBackgroundCodeResult).toHaveBeenCalledWith(
expect.objectContaining({
backgroundTask: expect.objectContaining({ taskId, status: 'error', cancelled: true }),
}),
);
expect(persistBackgroundCodeResult).toHaveBeenCalledWith(
expect.objectContaining({
reapply: true,
output: expect.stringContaining('cancellation requested'),
backgroundTask: expect.objectContaining({
taskId,
status: 'error',
cancelled: true,
}),
}),
);
expect(delayedWrites).toBe(0);
const reused = await runBatch(handler, {
toolCalls: [
{
id: 'call_reuse_bash',
name: 'bash_tool',
args: { command: 'pwd', run_in_background: true },
},
],
agentId: 'agent_cancel_bash',
configurable,
metadata: { ...metadata, run_id: 'reuse-run' },
});
const reusedTaskId = JSON.parse(reused[0].content).background_task_id as string;
await flushMicrotasks();
await flushMicrotasks();
const reusedTerminal = JSON.parse(
await runCheckBackgroundTask({
userId: 'exec_user',
conversationId: 'cancel_bash_conversation',
args: { background_task_id: reusedTaskId },
}),
);
expect(reusedTerminal).toMatchObject({ status: 'completed', result: '/workspace' });
expect(invocations).toBe(2);
});
it('blocks normalized arguments before registering or dispatching a background task', async () => {
const protectedValue = 'PROTECTED-BACKGROUND';
const state = { calls: 0 } as { calls: number; lastInput?: Record<string, unknown> };

View file

@ -244,6 +244,8 @@ export interface ToolExecuteOptions {
}>;
/** Trusted detached-subagent task scope for polling and parent controls. */
subagentTasks?: SubagentTaskConfig;
/** Trusted deployment gate for cooperative ordinary-tool cancellation. */
ordinaryToolCancellation?: boolean;
/** Callback to process tool artifacts (code output files, file citations, etc.) */
toolEndCallback?: ToolEndCallback;
/** Durable internal-completion adapter, present only for an Event Actor invocation. */
@ -5209,6 +5211,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
emitAttachment,
emitPtcProgress,
subagentTasks,
ordinaryToolCancellation = false,
provisionFiles,
} = options;
@ -5516,12 +5519,24 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}),
};
}
const backgroundAbortController = new AbortController();
let backgroundAbortSource: 'manual' | 'timeout' | undefined;
const created = backgroundTaskRegistry.create({
...(detachedReservation?.status === 'reserved'
? { taskId: detachedReservation.taskId }
: {}),
...registration,
...(capacityPermit == null ? {} : { capacityPermit }),
requestCancellation: () => {
if (backgroundAbortSource != null || backgroundAbortController.signal.aborted) {
return false;
}
backgroundAbortSource = 'manual';
backgroundAbortController.abort(
new DOMException('Background task cancellation requested', 'AbortError'),
);
return true;
},
});
if ('atCapacity' in created) {
if (detachedReservation?.status === 'reserved') {
@ -5587,7 +5602,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const persistBackgroundResult = async (params: {
output?: string;
artifact?: unknown;
status: 'completed' | 'error';
status: 'completed' | 'error' | 'cancelled';
}): Promise<void> => {
/** A provider id alone is not a durable part identity: it may
* repeat in later turns of the same response. New automatic
@ -5610,7 +5625,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
return {
taskId: task.id,
toolName: tc.name,
status: params.status,
status: params.status === 'cancelled' ? 'error' : params.status,
...(params.status === 'cancelled' ? { cancelled: true } : {}),
settledAt: new Date(current?.updatedAt ?? Date.now()),
...(completionPreregistered ? { completionWakeup: true } : {}),
...(current?.resultClaim != null
@ -5743,7 +5759,8 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
* overwrite it. */
dispatchedAt: task.createdAt,
codeExecutionContext,
...(detachedReservation?.status === 'reserved' || !completionPreregistered
...(detachedReservation?.status === 'reserved' ||
(!completionPreregistered && params.status !== 'cancelled')
? {}
: { backgroundTask, resolveBackgroundTask }),
output: params.output ?? localTask?.result,
@ -5816,7 +5833,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const persistSettledBackgroundResult = async (params: {
output?: string;
artifact?: unknown;
status: 'completed' | 'error';
status: 'completed' | 'error' | 'cancelled';
}): Promise<void> => {
if (harvestEnabled) {
await persistBackgroundResult(params);
@ -5838,7 +5855,6 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
};
let invokePromise: Promise<{ content?: unknown; artifact?: unknown }>;
const backgroundAbortController = new AbortController();
try {
invokePromise = Promise.resolve(
tool.invoke(normalizedArgs, {
@ -5966,6 +5982,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
};
let producerRetirementTimeout: ReturnType<typeof setTimeout> | undefined;
const requestBackgroundAbort = (): void => {
backgroundAbortSource ??= 'timeout';
backgroundAbortController.abort(
new DOMException('Background task timed out', 'AbortError'),
);
@ -6103,12 +6120,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
const neutralizedError = filteredError?.errorMessage ?? errorOutput;
const deliveredError = toBackgroundToolFailure(tc.name, neutralizedError);
const registryError = isCodeCall ? deliveredError : neutralizedError;
/** Only an owner-authorized request is cancellation evidence.
* Providers and timeout controllers also use AbortError, so
* classifying by error shape would turn failures into a false
* claim that the owner cancelled the task. */
const detachedTerminalStatus =
toolError instanceof Error &&
(toolError.name === 'AbortError' ||
(toolError as Error & { code?: string }).code === 'ABORT_ERR')
? 'cancelled'
: 'failed';
backgroundAbortSource === 'manual' ? 'cancelled' : 'failed';
if (
!(await persistDetachedTerminal({
status: detachedTerminalStatus,
@ -6117,19 +6134,30 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
) {
return;
}
backgroundTaskRegistry.fail(
backgroundUserId,
backgroundConversationId,
task.id,
registryError,
/** Failed code tasks join the heal path too: without this,
* a full-row save reverting the error patch would leave
* the dispatch card on the handle JSON forever. */
{ harvestStarted: harvestEnabled },
);
const settleOptions = { harvestStarted: harvestEnabled };
if (detachedTerminalStatus === 'cancelled') {
backgroundTaskRegistry.cancel(
backgroundUserId,
backgroundConversationId,
task.id,
registryError,
settleOptions,
);
} else {
backgroundTaskRegistry.fail(
backgroundUserId,
backgroundConversationId,
task.id,
registryError,
/** Failed code tasks join the heal path too: without this,
* a full-row save reverting the error patch would leave
* the dispatch card on the handle JSON forever. */
settleOptions,
);
}
await persistSettledBackgroundResult({
output: deliveredError,
status: 'error',
status: detachedTerminalStatus === 'cancelled' ? 'cancelled' : 'error',
});
await wakeDetachedActor();
} finally {
@ -6195,6 +6223,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
subagentTasks,
claimBackgroundToolResult: backgroundToolCompletion?.claim,
recoverDeadBackgroundToolClaim: backgroundToolCompletion?.recoverDeadClaim,
ordinaryToolCancellation,
});
const taskSnapshot = getBackgroundTaskSnapshot({
userId: backgroundUserId,
@ -6362,12 +6391,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
}
if (persistBackgroundCodeResult && delivery.messageId) {
/** Error tasks carry their message in `error`, not
/** Error/cancelled tasks carry their message in `error`, not
* `result`; abort-confirmed timeouts store it raw, so
* wrap here — `toBackgroundToolFailure` is a no-op for
* already-wrapped detached failures. */
const reapplyOutput =
delivery.status === 'error'
delivery.status === 'error' || delivery.status === 'cancelled'
? toBackgroundToolFailure(
delivery.toolName,
delivery.error ?? delivery.result ?? 'Background task failed',
@ -6382,6 +6411,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
agentId: delivery.agentId,
output: reapplyOutput,
attachments: delivery.attachments,
...(delivery.backgroundTask == null
? {}
: { backgroundTask: delivery.backgroundTask }),
reapply: true,
}).catch((reapplyError) => {
logger.warn(

View file

@ -33,7 +33,10 @@ export interface ProcessedCodeOutput {
export interface BackgroundToolResultState {
taskId: string;
toolName: string;
/** Cancelled executions retain the pre-existing durable `error` state so
* older replicas can still claim them during a rolling deploy. */
status: 'completed' | 'error';
cancelled?: true;
settledAt: Date;
/** This exact task owns a pre-registered automatic continuation delivery. */
completionWakeup?: true;

View file

@ -347,7 +347,10 @@ describe('agent background task config', () => {
if (!result.success) {
return;
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: true });
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: true,
ordinaryToolCancellation: false,
});
});
it('accepts an administrator poll-only policy', () => {
@ -360,7 +363,26 @@ describe('agent background task config', () => {
if (!result.success) {
return;
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({ completionWakeups: false });
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: false,
ordinaryToolCancellation: false,
});
});
it('accepts administrator-enabled ordinary tool cancellation', () => {
const result = configSchema.safeParse({
version: '1.0',
endpoints: { agents: { backgroundTasks: { ordinaryToolCancellation: true } } },
});
expect(result.success).toBe(true);
if (!result.success) {
return;
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: true,
ordinaryToolCancellation: true,
});
});
});

View file

@ -1336,6 +1336,9 @@ export const agentsEndpointSchema = baseEndpointSchema
backgroundTasks: z
.object({
completionWakeups: z.boolean().optional().default(true),
/** Cooperative cancellation for process-local ordinary tools. Off
* by default so existing deployments opt into the new control. */
ordinaryToolCancellation: z.boolean().optional().default(false),
})
.optional(),
skills: z

View file

@ -93,6 +93,8 @@ export namespace Agents {
taskId: string;
toolName: string;
status: 'completed' | 'error';
/** Additive rolling-deploy-compatible cancellation discriminator. */
cancelled?: true;
settledAt: Date;
resultClaim?: {
kind: 'manual' | 'wakeup';

View file

@ -1474,6 +1474,53 @@ describe('Message Operations', () => {
expect(claim.status).toBe('acquired');
});
it('claims a cancelled ordinary background-tool result as terminal evidence', async () => {
await saveMessage(mockCtx, {
...mockMessageData,
content: [
{
type: 'tool_call',
tool_call: {
id: 'call-cancelled-tool',
name: 'bash_tool',
output: 'Background task cancellation requested',
backgroundTask: {
version: 1,
taskId: 'task-cancelled-tool',
toolName: 'bash_tool',
status: 'error',
cancelled: true,
settledAt: new Date(),
completionWakeup: true,
},
},
},
],
});
await expect(
claimBackgroundToolResults({
userId: 'user123',
conversationId: mockMessageData.conversationId as string,
messageId: 'msg123',
taskId: 'task-cancelled-tool',
kind: 'wakeup',
claimId: 'delivery-cancelled-tool',
}),
).resolves.toEqual({
status: 'acquired',
results: [
{
taskId: 'task-cancelled-tool',
toolCallId: 'call-cancelled-tool',
toolName: 'bash_tool',
status: 'cancelled',
output: 'Background task cancellation requested',
},
],
});
});
it('elects one result consumer and batches terminal siblings for a wakeup', async () => {
const terminal = (id: string, taskId: string, output: string) => ({
type: 'tool_call',

View file

@ -459,7 +459,7 @@ export interface BackgroundToolResultRecord {
taskId: string;
toolCallId: string;
toolName: string;
status: 'completed' | 'error';
status: 'completed' | 'error' | 'cancelled';
output: string;
agentId?: string;
}
@ -647,6 +647,7 @@ export interface MessageMethods {
taskId: string;
toolName: string;
status: 'completed' | 'error';
cancelled?: true;
settledAt: Date;
completionWakeup?: true;
resultClaim?: {
@ -1144,6 +1145,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
taskId: string;
toolName: string;
status: 'completed' | 'error';
cancelled?: true;
settledAt: Date;
completionWakeup?: true;
resultClaim?: {
@ -1193,6 +1195,8 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
partPatch['content.$[part].tool_call.backgroundTask.taskId'] = backgroundTask.taskId;
partPatch['content.$[part].tool_call.backgroundTask.toolName'] = backgroundTask.toolName;
partPatch['content.$[part].tool_call.backgroundTask.status'] = backgroundTask.status;
partPatch['content.$[part].tool_call.backgroundTask.cancelled'] =
backgroundTask.cancelled === true;
partPatch['content.$[part].tool_call.backgroundTask.settledAt'] = backgroundTask.settledAt;
if (backgroundTask.completionWakeup === true) {
partPatch['content.$[part].tool_call.backgroundTask.completionWakeup'] = true;
@ -1401,6 +1405,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
taskId?: unknown;
toolName?: unknown;
status?: unknown;
cancelled?: unknown;
resultClaim?: { kind?: unknown; claimId?: unknown };
};
};
@ -1427,7 +1432,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
taskId: task.taskId,
toolCallId: toolCall.id,
toolName: task.toolName,
status: task.status,
status: task.cancelled === true ? 'cancelled' : task.status,
output: typeof toolCall.output === 'string' ? toolCall.output : '',
...(resultAgentId == null ? {} : { agentId: resultAgentId }),
});