📬 fix: Persist Background Results Before Parent Rows (#16101)

* 📬 fix: Persist Background Results Before Parent Rows

* fix: Harden durable background result delivery

* fix: Fence durable background completion ownership

* fix: reconcile durable background receipt ownership

* test: align background receipt mocks with contracts

* fix: Fence durable result ownership and recover interrupted cleanup

* fix: Mark independent receipt projections for task-local ownership

* fix: Carry receipt provenance through message patch inputs

* fix: Preserve resumed receipt delivery during mutual claim yielding

* fix: Preserve conversation cleanup for non-receipt owner identities

* fix: Preserve receipt recovery across cleanup and retry exhaustion
This commit is contained in:
Danny Avila 2026-09-19 20:26:58 -04:00 • committed by GitHub
parent 5d2fa85d1f
commit 8a3ab43965
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 1863 additions and 87 deletions

View file

@ -1355,7 +1355,10 @@ describe('OpenAIChatCompletionController', () => {
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
req.config.endpoints.agents.backgroundTasks = {
ordinaryToolCancellation: true,
completionResultMaxChars: 4096,
};
await OpenAIChatCompletionController(req, res);
const [initializeParams, dbMethods] = initializeAgent.mock.calls.at(-1);
@ -1384,6 +1387,7 @@ describe('OpenAIChatCompletionController', () => {
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal);
expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId);
const effectiveSignal = new AbortController().signal;

View file

@ -2065,7 +2065,10 @@ describe('createResponse controller', () => {
const { loadAgentTools, loadToolsForExecution } = require('~/server/services/ToolService');
const { filterFilesByAgentAccess } = require('~/server/services/Files/permissions');
req.config.endpoints.agents.backgroundTasks = { ordinaryToolCancellation: true };
req.config.endpoints.agents.backgroundTasks = {
ordinaryToolCancellation: true,
completionResultMaxChars: 4096,
};
req.body.stream = stream;
await createResponse(req, res);
@ -2095,6 +2098,7 @@ describe('createResponse controller', () => {
const toolExecuteOptions = createToolExecuteHandler.mock.calls.at(-1)[0];
expect(toolExecuteOptions.ordinaryToolCancellation).toBe(true);
expect(toolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
expect(toolExecuteOptions.runSignal).toBe(mockExecution.signal);
expect(toolExecuteOptions.foregroundRunId).toBe(initializeParams.requestBody.messageId);
const effectiveSignal = new AbortController().signal;

View file

@ -490,6 +490,8 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
const allowedProviders = new Set(agentsEConfig?.allowedProviders);
const ordinaryToolCancellationEnabled =
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
const backgroundCompletionResultMaxChars =
agentsEConfig?.backgroundTasks?.completionResultMaxChars;
// Create tool loader
const loadTools = createToolLoader({ req, res, signal: execution.signal });
@ -840,6 +842,7 @@ const executeOpenAIChatCompletion = async (envelope, { req, res }) => {
runSignal: execution.signal,
foregroundRunId: responseId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
backgroundCompletionResultMaxChars,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,

View file

@ -732,6 +732,8 @@ const executeResponse = async (envelope, { req, res }) => {
const agentsEConfig = appConfig?.endpoints?.[EModelEndpoint.agents];
const ordinaryToolCancellationEnabled =
agentsEConfig?.backgroundTasks?.ordinaryToolCancellation === true;
const backgroundCompletionResultMaxChars =
agentsEConfig?.backgroundTasks?.completionResultMaxChars;
const previousMessages = request.previous_response_id
? await loadPreviousMessages(request.previous_response_id, principal.userId)
: [];
@ -1194,6 +1196,7 @@ const executeResponse = async (envelope, { req, res }) => {
runSignal: execution.signal,
foregroundRunId: responseId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
backgroundCompletionResultMaxChars,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,
@ -1424,6 +1427,7 @@ const executeResponse = async (envelope, { req, res }) => {
runSignal: execution.signal,
foregroundRunId: responseId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
backgroundCompletionResultMaxChars,
provisionFiles: createProvisionFilesCallback({
req,
agentToolContexts,

View file

@ -86,6 +86,9 @@ module.exports = {
requeueAgentTrigger: service.requeue,
retireAgentTrigger: service.retire,
renewAgentTriggerProducerLease: service.renewProducerLease,
persistAgentBackgroundToolResult: service.persistBackgroundToolResult,
getAgentBackgroundToolResultClaim: service.getBackgroundToolResultClaim,
releaseAgentBackgroundToolResultClaims: service.releaseBackgroundToolResultClaims,
drainAgentTriggerDeliveriesForUser: service.drainUser,
prepareAgentTriggerUserPurge: service.prepareUserPurge,
cancelAgentTriggerUserPurge: service.cancelUserPurge,

View file

@ -2,9 +2,13 @@ const {
createBackgroundToolCompletionWakeupHandler,
createBackgroundToolDeadClaimRecovery,
createBackgroundToolResultHandler,
claimBackgroundToolResult: claimResult,
} = require('@librechat/api');
const {
enqueueAgentTrigger,
persistAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
releaseAgentBackgroundToolResultClaims,
renewAgentTriggerProducerLease,
retireAgentTrigger,
} = require('../../Agents/triggers');
@ -13,12 +17,17 @@ const preregisterBackgroundToolCompletion = createBackgroundToolCompletionWakeup
enqueueAgentTrigger,
retireAgentTrigger,
renewAgentTriggerProducerLease,
(deliveryKey, sourceId, result) =>
persistAgentBackgroundToolResult({ deliveryKey, sourceId, result }),
);
function createBackgroundToolResultPersistence({ req, updateToolCallResult }) {
return createBackgroundToolResultHandler({ req, updateToolCallResult });
}
const claimBackgroundToolResult = (db, input) =>
claimResult(db, getAgentBackgroundToolResultClaim, input);
function createDeadBackgroundToolClaimRecovery(
releaseBackgroundToolResultClaims,
getGenerationJob,
@ -29,11 +38,13 @@ function createDeadBackgroundToolClaimRecovery(
releaseBackgroundToolResultClaims,
getGenerationJob,
fenceGenerationClaim,
releaseAgentBackgroundToolResultClaims,
);
}
module.exports = {
preregisterBackgroundToolCompletion,
createBackgroundToolResultPersistence,
claimBackgroundToolResult,
createDeadBackgroundToolClaimRecovery,
};

View file

@ -95,6 +95,7 @@ const subagentThreadTaskStore = require('./subagentThreadStore');
const {
preregisterBackgroundToolCompletion,
createBackgroundToolResultPersistence,
claimBackgroundToolResult,
createDeadBackgroundToolClaimRecovery,
} = require('./backgroundCompletion');
const { logViolation } = require('~/cache');
@ -215,6 +216,8 @@ const initializeClientWithProvider = async ({
const ordinaryToolCancellationEnabled =
appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.ordinaryToolCancellation ===
true;
const backgroundCompletionResultMaxChars =
appConfig?.endpoints?.[EModelEndpoint.agents]?.backgroundTasks?.completionResultMaxChars;
/** The normal controller resolves this once for timestamp anchoring. Reuse
* that trusted document for child-thread execution policy; resume and direct
* callers fall back to the same owner-scoped lookup. */
@ -440,6 +443,7 @@ const initializeClientWithProvider = async ({
runSignal: signal,
foregroundRunId,
ordinaryToolCancellation: ordinaryToolCancellationEnabled,
backgroundCompletionResultMaxChars,
loadTools: async (
toolNames,
agentId,
@ -515,7 +519,7 @@ const initializeClientWithProvider = async ({
req,
updateToolCallResult: db.updateToolCallResult,
}),
claim: db.claimBackgroundToolResults,
claim: (input) => claimBackgroundToolResult(db, input),
recoverDeadClaim: createDeadBackgroundToolClaimRecovery(
db.releaseBackgroundToolResultClaims,
(conversationId) => GenerationJobManager.getJob(conversationId),

View file

@ -408,10 +408,11 @@ describe('initializeClient — processAgent ACL gate', () => {
endpointOption: makeEndpointOption(),
});
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(false);
expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBeUndefined();
const enabledReq = makeReq();
enabledReq.config.endpoints.agents = {
backgroundTasks: { ordinaryToolCancellation: true },
backgroundTasks: { ordinaryToolCancellation: true, completionResultMaxChars: 4096 },
};
await initializeClient({
req: enabledReq,
@ -420,6 +421,7 @@ describe('initializeClient — processAgent ACL gate', () => {
endpointOption: makeEndpointOption(),
});
expect(capturedToolExecuteOptions.ordinaryToolCancellation).toBe(true);
expect(capturedToolExecuteOptions.backgroundCompletionResultMaxChars).toBe(4096);
});
it('propagates an expected-MCP-tools failure from the runtime tool loader', async () => {

View file

@ -0,0 +1,82 @@
import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas';
import { claimBackgroundToolResult } from './backgroundClaims';
describe('manual background receipt arbitration', () => {
const input = {
userId: 'user',
conversationId: 'conversation',
taskId: 'task',
kind: 'manual' as const,
claimId: 'poll',
};
const receipt = { kind: 'wakeup' as const, claimId: 'delivery', claimedAt: new Date() };
function fixture() {
const methods = {
claimBackgroundToolResults: jest
.fn<
ReturnType<MessageMethods['claimBackgroundToolResults']>,
Parameters<MessageMethods['claimBackgroundToolResults']>
>()
.mockResolvedValue({
status: 'acquired',
messageId: 'recovered-parent',
results: [
{
taskId: 'task',
toolCallId: 'call',
toolName: 'tool',
status: 'completed',
output: 'done',
},
],
}),
releaseBackgroundToolResultClaims: jest.fn(async () => true),
};
const lookup = jest
.fn<
ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>,
Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>
>()
.mockResolvedValue(null);
return { methods, lookup };
}
it('uses the recovered parent identity when the process-local registry is gone', async () => {
const { methods, lookup } = fixture();
lookup.mockResolvedValue(receipt);
await expect(claimBackgroundToolResult(methods, lookup, input)).resolves.toEqual({
status: 'claimed',
claim: receipt,
messageId: 'recovered-parent',
});
expect(lookup).toHaveBeenCalledWith(
expect.objectContaining({ parentMessageId: 'recovered-parent' }),
);
expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
expect.objectContaining({ messageId: 'recovered-parent', taskIds: ['task'] }),
);
});
it('releases a speculative message claim if receipt lookup fails', async () => {
const { methods, lookup } = fixture();
lookup.mockRejectedValue(new Error('database unavailable'));
await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow(
'database unavailable',
);
expect(methods.releaseBackgroundToolResultClaims).toHaveBeenCalledTimes(1);
});
it('does not report arbitration as complete if rollback fails', async () => {
const { methods, lookup } = fixture();
lookup.mockResolvedValue(receipt);
methods.releaseBackgroundToolResultClaims.mockResolvedValue(false);
await expect(claimBackgroundToolResult(methods, lookup, input)).rejects.toThrow(
'could not be released',
);
});
it('does not claim the projection when its known receipt already has an owner', async () => {
const { methods, lookup } = fixture();
lookup.mockResolvedValue(receipt);
await expect(
claimBackgroundToolResult(methods, lookup, { ...input, messageId: 'parent' }),
).resolves.toMatchObject({ status: 'claimed', messageId: 'parent' });
expect(methods.claimBackgroundToolResults).not.toHaveBeenCalled();
});
});

View file

@ -0,0 +1,58 @@
import type { AgentTriggerDeliveryMethods, MessageMethods } from '@librechat/data-schemas';
import { BACKGROUND_TOOL_COMPLETION_SOURCE } from './backgroundCompletionWakeup';
/** Reconciles manual polling with the independent automatic-delivery receipt,
* including polls reconstructed after the process-local registry was lost. */
export async function claimBackgroundToolResult(
methods: Pick<MessageMethods, 'claimBackgroundToolResults' | 'releaseBackgroundToolResultClaims'>,
getReceiptClaim: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'],
input: Parameters<MessageMethods['claimBackgroundToolResults']>[0],
): ReturnType<MessageMethods['claimBackgroundToolResults']> {
const lookup = (messageId: string) =>
getReceiptClaim({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId: input.userId,
conversationId: input.conversationId,
parentMessageId: messageId,
taskId: input.taskId,
});
if (input.messageId != null) {
const receiptClaim = await lookup(input.messageId);
if (receiptClaim != null && receiptClaim.claimId !== input.claimId) {
return { status: 'claimed', claim: receiptClaim, messageId: input.messageId };
}
}
const messageClaim = await methods.claimBackgroundToolResults(input);
if (messageClaim.status !== 'acquired') {
return messageClaim;
}
const messageId = input.messageId ?? messageClaim.messageId;
if (messageId == null) {
throw new Error('The background result claim has no parent message identity');
}
const release = async () => {
const released = await methods.releaseBackgroundToolResultClaims({
userId: input.userId,
conversationId: input.conversationId,
messageId,
taskIds: messageClaim.results.map((result) => result.taskId),
kind: input.kind,
claimId: input.claimId,
});
if (!released) {
throw new Error('The background result claim could not be released');
}
};
let receiptClaim: Awaited<ReturnType<typeof getReceiptClaim>>;
try {
receiptClaim = await lookup(messageId);
} catch (error) {
await release();
throw error;
}
if (receiptClaim == null || receiptClaim.claimId === input.claimId) {
return messageClaim;
}
await release();
return { status: 'claimed', claim: receiptClaim, messageId };
}

View file

@ -31,6 +31,13 @@ export interface BackgroundToolWakeupRetireOptions {
export interface BackgroundToolWakeupAdmission {
/** Renews durable proof that the process-local executor still owns work. */
renew: () => Promise<boolean>;
/** Persists terminal output on the pre-admitted delivery before the parent
* message projection exists. */
persistResult?: (result: {
status: 'completed' | 'error' | 'cancelled';
output: string;
settledAt: Date;
}) => Promise<boolean>;
/** Retires a delivery whose terminal result can no longer be made durable.
* Manual polling requires an atomic unclaimed-only transition: once a
* resolver owns a lease, its prepared continuation cannot be cancelled. */

View file

@ -1,4 +1,4 @@
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
import type { AgentTriggerProducerLeaseStatus } from '@librechat/data-schemas';
import type { EnqueueBackgroundToolCompletion } from './backgroundCompletionWakeup';
import {
@ -91,6 +91,10 @@ function resolverMethods() {
leaseUntil: new Date(NOW + 30_000),
}),
),
getAgentBackgroundToolResult: jest.fn(async () => null),
getAgentBackgroundToolResultClaim: jest.fn(async () => null),
claimAgentBackgroundToolResults: jest.fn(async () => ({ status: 'not_ready' as const })),
releaseAgentBackgroundToolResultClaims: jest.fn(async () => true),
},
};
}
@ -111,7 +115,13 @@ describe('background tool completion wakeups', () => {
>(async () => ({ deliveryKey: 'delivery-key-1' }));
const retire = jest.fn(async () => true);
const renew = jest.fn(async () => true);
const notify = createBackgroundToolCompletionWakeupHandler(enqueue, retire, renew);
const persistResult = jest.fn(async () => true);
const notify = createBackgroundToolCompletionWakeupHandler(
enqueue,
retire,
renew,
persistResult,
);
const admission = await notify(registration());
expect(admission).not.toBe(false);
@ -134,7 +144,7 @@ describe('background tool completion wakeups', () => {
expect(options).toEqual({
orderingKey: 'background-tool-completion:conversation-1:task-1',
availableAt: new Date(NOW + 250),
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: new Date(NOW + 30_000),
});
if (admission !== false) {
@ -145,6 +155,20 @@ describe('background tool completion wakeups', () => {
'background-tool-completion',
new Date(NOW + 30_000),
);
if (admission !== false) {
await expect(
admission.persistResult?.({
status: 'completed',
output: 'done',
settledAt: new Date(NOW),
}),
).resolves.toBe(true);
}
expect(persistResult).toHaveBeenCalledWith('delivery-key-1', 'background-tool-completion', {
status: 'completed',
output: 'done',
settledAt: new Date(NOW),
});
if (admission !== false) {
await expect(admission.retire('result unavailable')).resolves.toBe(true);
}
@ -180,6 +204,7 @@ describe('background tool completion wakeups', () => {
it('retires the batch-root delivery before releasing all of its sibling claims', async () => {
const retire = jest.fn(async () => true);
const release = jest.fn(async () => true);
const releaseReceipts = jest.fn(async () => true);
const getGenerationJob = jest.fn(async () => ({ status: 'complete' }));
const fenceGenerationClaim = jest.fn(async () => 'fenced' as const);
const recover = createBackgroundToolDeadClaimRecovery(
@ -187,6 +212,7 @@ describe('background tool completion wakeups', () => {
release,
getGenerationJob,
fenceGenerationClaim,
releaseReceipts,
);
await expect(
@ -218,6 +244,13 @@ describe('background tool completion wakeups', () => {
kind: 'wakeup',
claimId: 'batch-root-delivery',
});
expect(releaseReceipts).toHaveBeenCalledWith({
sourceId: 'background-tool-completion',
userId: 'user-1',
conversationId: 'conversation-1',
parentMessageId: 'response-1',
claimId: 'batch-root-delivery',
});
});
it('keeps a manual result claim while its owning generation is active', async () => {
@ -428,6 +461,120 @@ describe('background tool completion wakeups', () => {
);
});
it('continues from the independent delivery receipt before the parent projection lands', async () => {
const { methods } = resolverMethods();
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'not_ready', results: [] });
methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
status: 'acquired',
results: [
{
taskId: 'task-1',
toolCallId: 'call-1',
toolName: 'slow_tool',
status: 'completed',
output: 'independently durable',
},
],
} as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
const prepared = await resolve(await envelope(), { idempotencyKey: 'delivery-1' });
expect(prepared).toMatchObject({ status: 'ready', parentMessageId: 'response-2' });
expect(prepared?.status === 'ready' && prepared.input).toContain('independently durable');
expect(methods.claimAgentBackgroundToolResults).toHaveBeenCalledWith({
deliveryKey: 'delivery-1',
sourceId: 'background-tool-completion',
userId: 'user-1',
conversationId: 'conversation-1',
parentMessageId: 'response-1',
agentId: 'agent_parent_1',
claimId: 'delivery-1',
limit: 1,
});
expect(methods.claimBackgroundToolResults).toHaveBeenCalledWith(
expect.objectContaining({ taskId: 'task-1', kind: 'wakeup', claimId: 'delivery-1' }),
);
});
it('honors a manual message claim before an independent receipt', async () => {
const { methods } = resolverMethods();
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] });
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).resolves.toEqual({
status: 'settled',
});
expect(methods.claimAgentBackgroundToolResults).not.toHaveBeenCalled();
expect(methods.getAgentBackgroundToolResult).not.toHaveBeenCalled();
});
it('does not settle a resumed receipt owner while a speculative manual message claim yields', async () => {
const { methods } = resolverMethods();
methods.claimBackgroundToolResults.mockResolvedValueOnce({ status: 'claimed', results: [] });
methods.getAgentBackgroundToolResultClaim.mockResolvedValueOnce({
kind: 'wakeup',
claimId: 'delivery-1',
claimedAt: new Date(NOW),
} as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
{
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
deferWithoutAttempt: true,
},
);
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
expect.objectContaining({ claimId: 'delivery-1' }),
);
expect(methods.releaseBackgroundToolResultClaims).not.toHaveBeenCalled();
});
it('retries after yielding to a racing manual claim so mutual yielding cannot lose output', async () => {
const { methods } = resolverMethods();
methods.claimBackgroundToolResults
.mockResolvedValueOnce({ status: 'not_ready' } as never)
.mockResolvedValueOnce({
status: 'claimed',
claim: { kind: 'manual', claimId: 'poll-1' },
} as never);
methods.claimAgentBackgroundToolResults.mockResolvedValueOnce({
status: 'acquired',
results: [
{
taskId: 'task-1',
toolCallId: 'call-1',
toolName: 'slow_tool',
status: 'completed',
output: 'durable',
},
],
} as never);
const resolve = createBackgroundToolCompletionWakeupResolver({
methods: methods as never,
getGenerationJob: async () => null,
});
await expect(resolve(await envelope(), { idempotencyKey: 'delivery-1' })).rejects.toMatchObject(
{
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
deferWithoutAttempt: true,
},
);
expect(methods.releaseAgentBackgroundToolResultClaims).toHaveBeenCalledWith(
expect.objectContaining({ claimId: 'delivery-1', parentMessageId: 'response-1' }),
);
});
it('shares one bounded input budget across a full sibling batch', async () => {
const { methods } = resolverMethods();
const results = Array.from({ length: 8 }, (_, index) => ({
@ -468,7 +615,10 @@ describe('background tool completion wakeups', () => {
await prepared.releaseOnDefiniteFailure?.();
}
expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
expect.objectContaining({ taskIds: ['task-1', 'task-2'], claimId: 'delivery-1' }),
expect.objectContaining({ claimId: 'delivery-1', kind: 'wakeup' }),
);
expect(releaseBackgroundToolResultClaims).toHaveBeenCalledWith(
expect.not.objectContaining({ taskIds: expect.anything() }),
);
});

View file

@ -1,8 +1,9 @@
import { randomUUID } from 'node:crypto';
import { isEphemeralAgentId } from 'librechat-data-provider';
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 } from '@librechat/data-schemas';
import { AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 } from '@librechat/data-schemas';
import type {
AgentTriggerProducerLeaseStatus,
AgentTriggerDeliveryMethods,
BackgroundToolResultClaim,
ConversationMethods,
IMessage,
@ -51,6 +52,16 @@ export type RenewBackgroundToolCompletionProducerLease = (
leaseUntil: Date,
) => Promise<boolean>;
export type PersistBackgroundToolCompletionResult = (
deliveryKey: string,
sourceId: string,
result: {
status: 'completed' | 'error' | 'cancelled';
output: string;
settledAt: Date;
},
) => Promise<boolean>;
type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
Pick<MessageMethods, 'getMessages'> & {
claimBackgroundToolResults(params: {
@ -76,6 +87,14 @@ type WakeupMethods = Pick<ConversationMethods, 'getConvo'> &
sourceId: string;
now: Date;
}): Promise<AgentTriggerProducerLeaseStatus>;
getAgentBackgroundToolResult?(params: { deliveryKey: string; sourceId: string }): Promise<{
status: 'completed' | 'error' | 'cancelled';
output: string;
settledAt: Date;
} | null>;
claimAgentBackgroundToolResults?: AgentTriggerDeliveryMethods['claimAgentBackgroundToolResults'];
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
};
interface GenerationState {
@ -326,8 +345,33 @@ export function createBackgroundToolCompletionWakeupResolver({
agentId: envelope.target.agentId,
kind: 'wakeup',
claimId: context.idempotencyKey,
limit: 1,
});
if (claim.status === 'claimed') {
const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
taskId: registration.taskId,
});
if (receiptOwner?.claimId === context.idempotencyKey) {
const released = await methods.releaseAgentBackgroundToolResultClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
claimId: context.idempotencyKey,
});
if (released === false)
throw new Error('Background receipt claim release was not confirmed');
throw executionError('Background result ownership is being reconciled.', {
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
}
return { status: 'settled' };
}
if (claim.status === 'outcome_unknown') {
@ -336,47 +380,17 @@ export function createBackgroundToolCompletionWakeupResolver({
retryable: false,
});
}
if (claim.status !== 'acquired') {
let producerLease: AgentTriggerProducerLeaseStatus;
try {
producerLease = await methods.getAgentTriggerDeliveryProducerLease({
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
now: new Date(),
});
} catch (error) {
throw executionError(
`Background tool producer liveness is temporarily unavailable: ${
error instanceof Error ? error.message : String(error)
}`,
{ code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
);
}
if (producerLease.status === 'expired') {
throw executionError('The process-local background tool executor was lost.', {
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
retryable: false,
});
}
/** A live lease proves the invocation or its durable persistence retry
* still has an owner. Missing remains defer-only for compatibility with
* completion rows admitted before producer leases existed. */
throw executionError('The background tool result is not durable yet.', {
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
retryable: true,
status: 409,
retryAfter: '1',
deferWithoutAttempt: true,
if (claim.status === 'acquired') {
const taskIds = claim.results.map((result) => result.taskId);
const receiptOwner = await methods.getAgentBackgroundToolResultClaim?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
taskId: registration.taskId,
});
}
const taskIds = claim.results.map((result) => result.taskId);
const input = buildWakeupInput(claim.results);
return {
status: 'ready',
parentMessageId,
input,
releaseOnDefiniteFailure: async () => {
await methods.releaseBackgroundToolResultClaims({
if (receiptOwner != null && receiptOwner.claimId !== context.idempotencyKey) {
const released = await methods.releaseBackgroundToolResultClaims({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
@ -384,8 +398,153 @@ export function createBackgroundToolCompletionWakeupResolver({
kind: 'wakeup',
claimId: context.idempotencyKey,
});
},
};
if (!released) throw new Error('Background projection claim release was not confirmed');
return { status: 'settled' };
}
const input = buildWakeupInput(claim.results);
return {
status: 'ready',
parentMessageId,
input,
releaseOnDefiniteFailure: async () => {
const released = await methods.releaseBackgroundToolResultClaims({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
kind: 'wakeup',
claimId: context.idempotencyKey,
});
if (!released) throw new Error('Background projection claim release was not confirmed');
const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
claimId: context.idempotencyKey,
});
if (receiptReleased === false)
throw new Error('Background receipt claim release was not confirmed');
},
};
}
const receiptClaim = await methods.claimAgentBackgroundToolResults?.({
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
agentId: envelope.target.agentId,
claimId: context.idempotencyKey,
limit: 1,
});
if (receiptClaim?.status === 'claimed') {
return { status: 'settled' };
}
if (receiptClaim?.status === 'acquired') {
/** Reconcile with a parent projection that may have appeared while the
* receipt CAS was in flight. A manual poll that already owns the message
* wins; otherwise stamp this same automatic owner before dispatch. */
const projectedClaim = await methods.claimBackgroundToolResults({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
taskId: registration.taskId,
agentId: envelope.target.agentId,
kind: 'wakeup',
claimId: context.idempotencyKey,
limit: 1,
});
if (projectedClaim.status === 'claimed') {
const released = await methods.releaseAgentBackgroundToolResultClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
claimId: context.idempotencyKey,
});
if (released === false)
throw new Error('Background receipt claim release was not confirmed');
throw executionError('Background result ownership is being reconciled.', {
code: 'BACKGROUND_TOOL_CLAIM_RECONCILING',
retryable: true,
deferWithoutAttempt: true,
retryAfter: '1',
});
}
return {
status: 'ready',
parentMessageId,
input: buildWakeupInput(receiptClaim.results),
releaseOnDefiniteFailure: async () => {
const projectionReleased = await methods.releaseBackgroundToolResultClaims({
userId,
conversationId: envelope.target.conversationId,
messageId: envelope.target.parentMessageId,
kind: 'wakeup',
claimId: context.idempotencyKey,
});
if (projectedClaim.status === 'acquired' && !projectionReleased) {
throw new Error('Background projection claim release was not confirmed');
}
const receiptReleased = await methods.releaseAgentBackgroundToolResultClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId: envelope.target.conversationId,
parentMessageId: envelope.target.parentMessageId,
claimId: context.idempotencyKey,
});
if (receiptReleased === false)
throw new Error('Background receipt claim release was not confirmed');
},
};
}
const receipt =
methods.claimAgentBackgroundToolResults == null
? await methods.getAgentBackgroundToolResult?.({
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
})
: null;
if (receipt != null) {
return {
status: 'ready',
parentMessageId,
input: buildWakeupInput([
{ ...registration, status: receipt.status, output: receipt.output },
]),
};
}
let producerLease: AgentTriggerProducerLeaseStatus;
try {
producerLease = await methods.getAgentTriggerDeliveryProducerLease({
deliveryKey: context.idempotencyKey,
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
now: new Date(),
});
} catch (error) {
throw executionError(
`Background tool producer liveness is temporarily unavailable: ${
error instanceof Error ? error.message : String(error)
}`,
{ code: 'BACKGROUND_TOOL_PRODUCER_STATE_UNAVAILABLE', retryable: true },
);
}
if (producerLease.status === 'expired') {
throw executionError('The process-local background tool executor was lost.', {
code: 'BACKGROUND_TOOL_PRODUCER_LOST',
retryable: false,
});
}
/** A live lease proves the invocation or its durable persistence retry
* still has an owner. Missing remains defer-only for compatibility with
* completion rows admitted before producer leases existed. */
throw executionError('The background tool result is not durable yet.', {
code: 'BACKGROUND_TOOL_RESULT_NOT_READY',
retryable: true,
status: 409,
retryAfter: '1',
deferWithoutAttempt: true,
});
};
}
@ -394,6 +553,7 @@ export function createBackgroundToolCompletionWakeupHandler(
enqueue: EnqueueBackgroundToolCompletion,
retire: RetireBackgroundToolCompletion,
renewProducerLease: RenewBackgroundToolCompletionProducerLease,
persistResult?: PersistBackgroundToolCompletionResult,
): (
registration: BackgroundToolWakeupRegistration,
) => Promise<BackgroundToolWakeupAdmission | false> {
@ -437,7 +597,7 @@ export function createBackgroundToolCompletionWakeupHandler(
availableAt: new Date(
Math.max(Date.now(), registration.createdAt) + WAKEUP_ADMISSION_DELAY_MS,
),
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
});
return {
@ -447,6 +607,12 @@ export function createBackgroundToolCompletionWakeupHandler(
BACKGROUND_TOOL_COMPLETION_SOURCE,
new Date(Date.now() + BACKGROUND_TOOL_PRODUCER_LEASE_MS),
),
...(persistResult == null
? {}
: {
persistResult: (result) =>
persistResult(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, result),
}),
retire: (reason, options) =>
options == null
? retire(admitted.deliveryKey, BACKGROUND_TOOL_COMPLETION_SOURCE, reason)
@ -469,6 +635,7 @@ export function createBackgroundToolDeadClaimRecovery(
conversationId: string;
claimId: string;
}) => Promise<'fenced' | 'started' | 'unavailable'>,
releaseReceiptClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'],
): BackgroundToolDeadClaimRecovery {
return async ({ userId, conversationId, messageId, claimId, kind, generationId }) => {
if (kind === 'manual') {
@ -518,12 +685,24 @@ export function createBackgroundToolDeadClaimRecovery(
if (await claimGenerationIsActive()) {
return false;
}
return releaseClaims({
const released = await releaseClaims({
userId,
conversationId,
messageId,
kind: 'wakeup',
claimId,
});
if (!released) {
return false;
}
return (
(await releaseReceiptClaims?.({
sourceId: BACKGROUND_TOOL_COMPLETION_SOURCE,
userId,
conversationId,
parentMessageId: messageId,
claimId,
})) ?? true
);
};
}

View file

@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas';
import { tool as structuredTool } from '@librechat/agents/langchain/tools';
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { FiltersConfig } from 'librechat-data-provider';
import type { BackgroundToolWakeupAdmission } from './backgroundCompletion';
import type { ToolExecuteOptions } from './handlers';
import {
BACKGROUND_TASK_ABORT_GRACE_MS,
@ -147,9 +148,13 @@ describe('createToolExecuteHandler — background tool calls', () => {
it('pre-registers an ordinary completion before invoke and persists its terminal receipt', async () => {
const events: string[] = [];
const retire = jest.fn(async () => true);
const persistResult = jest.fn(async () => {
events.push('receipt');
return true;
});
const preregister = jest.fn(async () => {
events.push('preregister');
return { renew: jest.fn(async () => true), retire };
return { renew: jest.fn(async () => true), persistResult, retire };
});
const persist = jest.fn(async () => {
events.push('persist');
@ -198,7 +203,7 @@ describe('createToolExecuteHandler — background tool calls', () => {
);
expect(JSON.parse(dispatch.content).message).toContain('host will resume you');
await flushMicrotasks();
expect(events).toEqual(['preregister', 'invoke', 'persist']);
expect(events).toEqual(['preregister', 'invoke', 'receipt', 'persist']);
expect(preregister).toHaveBeenCalledWith(
expect.objectContaining({
toolCallId: 'call-wakeup',
@ -217,6 +222,9 @@ describe('createToolExecuteHandler — background tool calls', () => {
}),
}),
);
expect(persistResult).toHaveBeenCalledWith(
expect.objectContaining({ status: 'completed', output: 'durable result' }),
);
expect(retire).not.toHaveBeenCalled();
});
@ -264,6 +272,55 @@ describe('createToolExecuteHandler — background tool calls', () => {
);
});
it.each(['acknowledged', 'ambiguous'])(
'keeps delivery live for an %s receipt without a projection',
async (ack) => {
const retire = jest.fn(async () => true);
const persistResult = jest.fn(async () => {
if (ack === 'ambiguous') throw new Error('receipt committed but acknowledgement lost');
return true;
});
const tool = {
name: 'search_mcp_docs',
description: 'search docs',
schema: z.object({ q: z.string() }),
invoke: jest.fn(async () => ({ content: 'available before parent row' })),
} as unknown as StructuredToolInterface;
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({
renew: jest.fn(async () => true),
persistResult,
retire,
})),
persist: jest.fn(async () => false),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [
{
id: `call-independent-receipt-${ack}`,
name: tool.name,
args: { q: 'receipt', run_in_background: true },
stepId: 'step-independent-receipt',
},
],
agentId: 'agent_parent_1',
configurable: buildConfig([tool.name]),
metadata: { thread_id: 'exec_convo', run_id: `response-independent-receipt-${ack}` },
});
await flushMicrotasks();
expect(persistResult).toHaveBeenCalledWith(
expect.objectContaining({ status: 'completed', output: 'available before parent row' }),
);
expect(retire).not.toHaveBeenCalled();
},
);
it('keeps polling guidance when the completion adapter skips registration', async () => {
const tool = makeSearchTool({ calls: 0 });
const preregister = jest.fn(async () => false as const);
@ -2570,6 +2627,84 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
expect(poll[0].artifact).toEqual(CODE_ARTIFACT);
});
it('does not publish an output-only receipt when generated files miss their message anchor', async () => {
const state: CodeToolState = { calls: 0 };
const retire = jest.fn(async () => true);
const persistResult = jest.fn(async () => true);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [makeCodeTool(state)] }),
persistBackgroundCodeResult: async () => ({
deliveryReady: false,
attachments: [{ filename: 'plot.png' }],
}),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({
renew: jest.fn(async () => true),
persistResult,
retire,
})),
persist: jest.fn(async () => true),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [codeCall({ id: 'call_code_missing_anchor' })],
agentId: 'a',
configurable: buildConfig(['execute_code']),
metadata: { thread_id: 'exec_convo_code_missing_anchor', run_id: 'msg-missing-anchor' },
});
await flushMicrotasks();
await flushMicrotasks();
expect(persistResult).not.toHaveBeenCalled();
expect(retire).toHaveBeenCalledWith('background code result was not persisted', undefined);
});
it('uses the configured durable completion result size', async () => {
const persistResult: jest.MockedFunction<
NonNullable<BackgroundToolWakeupAdmission['persistResult']>
> = jest.fn(
async (_result: Parameters<NonNullable<BackgroundToolWakeupAdmission['persistResult']>>[0]) =>
true,
);
const tool = {
name: 'execute_code',
description: 'run code',
schema: z.object({ lang: z.string(), code: z.string() }),
invoke: jest.fn(async () => ({ content: '0123456789'.repeat(10), artifact: CODE_ARTIFACT })),
} as unknown as StructuredToolInterface;
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [tool] }),
backgroundCompletionResultMaxChars: 32,
persistBackgroundCodeResult: async () => ({ deliveryReady: true, attachments: [] }),
backgroundToolCompletion: {
preregister: jest.fn(async () => ({
renew: jest.fn(async () => true),
persistResult,
retire: jest.fn(async () => true),
})),
persist: jest.fn(async () => true),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
});
await runBatch(handler, {
toolCalls: [codeCall({ id: 'call-sized-receipt' })],
agentId: 'agent_parent_1',
configurable: buildConfig([tool.name]),
metadata: { thread_id: 'exec_convo', run_id: 'response-sized-receipt' },
});
await flushMicrotasks();
await flushMicrotasks();
await flushMicrotasks();
expect(persistResult).toHaveBeenCalledWith(
expect.objectContaining({ status: 'completed', output: expect.any(String) }),
);
expect(persistResult.mock.calls[0][0].output).toHaveLength(32);
});
it('makes a completion-time generated-file policy rejection terminal across polls', async () => {
const protectedValue = 'PROTECTED-GENERATED-FILE-BYTES';
const blockedArtifact = {
@ -2596,12 +2731,17 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
});
const toolEndCallback = jest.fn();
const retire = jest.fn(async () => true);
const persistResult = jest.fn(async () => true);
const handler = createToolExecuteHandler({
loadTools: async () => ({ loadedTools: [codeTool] }),
toolEndCallback,
persistBackgroundCodeResult,
backgroundToolCompletion: {
preregister: jest.fn(async () => ({ renew: jest.fn(async () => true), retire })),
preregister: jest.fn(async () => ({
renew: jest.fn(async () => true),
retire,
persistResult,
})),
persist: jest.fn(async () => true),
claim: jest.fn(async () => ({ status: 'acquired' as const, results: [] })),
},
@ -2621,7 +2761,10 @@ describe('createToolExecuteHandler — backgrounded code execution', () => {
const taskId = JSON.parse(dispatch[0].content).background_task_id;
await flushMicrotasks();
await flushMicrotasks();
expect(retire).toHaveBeenCalledWith('background code result persistence failed', undefined);
expect(persistResult).toHaveBeenCalledWith(
expect.objectContaining({ status: 'error', output: MODEL_BOUND_FILE_CONTENT_BLOCK }),
);
expect(retire).not.toHaveBeenCalled();
await flushMicrotasks();
for (const pollId of ['call_policy_poll_1', 'call_policy_poll_2']) {

View file

@ -3,6 +3,7 @@ import { Types } from 'mongoose';
import { GraphEvents, Constants, ToolEndHandler } from '@librechat/agents';
import { logger, normalizeSkillFrontmatterKeys } from '@librechat/data-schemas';
import {
AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT,
hasActivePiiFields,
hasActivePiiPatterns,
hasToolCallErrorPrefix,
@ -271,6 +272,8 @@ export interface ToolExecuteOptions {
runFiles?: Pick<RunFileSession, 'isActive' | 'prepareTools' | 'withCodeExecution'>;
/** Trusted deployment gate for cooperative ordinary-tool cancellation. */
ordinaryToolCancellation?: boolean;
/** Deployment-configured cap for terminal output copied into durable receipts. */
backgroundCompletionResultMaxChars?: number;
/** Callback to process tool artifacts (code output files, file citations, etc.) */
toolEndCallback?: ToolEndCallback;
/** Durable internal-completion adapter, present only for an Event Actor invocation. */
@ -5317,6 +5320,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
subagentTasks,
runFiles,
ordinaryToolCancellation = false,
backgroundCompletionResultMaxChars = AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT,
provisionFiles,
} = options;
@ -5774,6 +5778,9 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
...(params.status === 'cancelled' ? { cancelled: true } : {}),
settledAt: new Date(current?.updatedAt ?? Date.now()),
...(completionPreregistered ? { completionWakeup: true } : {}),
...(completionAdmission?.persistResult != null
? { completionReceipt: true }
: {}),
...(current?.resultClaim != null
? {
resultClaim: {
@ -5794,10 +5801,40 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
task.id,
);
const backgroundTask = resolveBackgroundTask();
let durableReceiptReady = false;
let durableReceiptAmbiguous = false;
const persistDurableReceipt = async (receipt: {
status: 'completed' | 'error' | 'cancelled';
output?: string;
}): Promise<boolean> => {
if (completionAdmission?.persistResult == null) {
return false;
}
try {
return await completionAdmission.persistResult({
status: receipt.status,
output: truncateMiddle(
receipt.output ?? '',
backgroundCompletionResultMaxChars,
),
settledAt: backgroundTask.settledAt,
});
} catch (receiptError) {
durableReceiptAmbiguous = true;
logger.warn(
`[background] Failed to persist independent result receipt for task ${task.id}:`,
receiptError,
);
return false;
}
};
const retireFailedPersistence = async (
reason: string,
certainty: 'definite' | 'ambiguous',
): Promise<void> => {
if (durableReceiptAmbiguous) {
return;
}
if (completionAdmission == null) {
backgroundTaskRegistry.markCompletionPersistenceFailed(
backgroundUserId,
@ -5852,6 +5889,10 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
};
if (!harvestEnabled || !persistBackgroundCodeResult) {
durableReceiptReady = await persistDurableReceipt({
status: params.status,
output: params.output ?? localTask?.result,
});
if (
backgroundToolCompletion == null ||
detachedReservation?.status === 'reserved'
@ -5870,17 +5911,19 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
backgroundTask,
resolveBackgroundTask,
});
if (!deliveryReady) {
if (!deliveryReady && !durableReceiptReady) {
await retireFailedPersistence(
'background tool result was not persisted',
'definite',
);
}
} catch (persistError) {
await retireFailedPersistence(
'background tool result persistence failed',
isContentFilterError(persistError) ? 'definite' : 'ambiguous',
);
if (!durableReceiptReady) {
await retireFailedPersistence(
'background tool result persistence failed',
isContentFilterError(persistError) ? 'definite' : 'ambiguous',
);
}
logger.warn(
`[background] Failed to persist result for task ${task.id}:`,
persistError,
@ -5922,7 +5965,7 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
task.id,
params.artifact,
);
if (completionPreregistered) {
if (completionPreregistered && !durableReceiptReady) {
await retireFailedPersistence(
'background code result had no durable message anchor',
'definite',
@ -5930,6 +5973,12 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
}
return;
}
if (persisted.deliveryReady !== false) {
durableReceiptReady = await persistDurableReceipt({
status: params.status,
output: params.output ?? localTask?.result,
});
}
if (persisted.deliveryReady === false) {
await retireFailedPersistence(
'background code result was not persisted',
@ -5943,26 +5992,38 @@ export function createToolExecuteHandler(options: ToolExecuteOptions): EventHand
persisted.attachments,
);
} catch (persistError) {
if (completionPreregistered) {
await retireFailedPersistence(
'background code result persistence failed',
isContentFilterError(persistError) ? 'definite' : 'ambiguous',
);
}
if (isContentFilterError(persistError)) {
const blockedMessage =
persistError instanceof ContentFilterError
? modelBoundContentFilterErrorMessage(persistError.body)
: persistError.body.message;
backgroundTaskRegistry.blockArtifact(
backgroundUserId,
backgroundConversationId,
task.id,
persistError instanceof ContentFilterError
? modelBoundContentFilterErrorMessage(persistError.body)
: persistError.body.message,
blockedMessage,
);
durableReceiptReady = await persistDurableReceipt({
status: 'error',
output: blockedMessage,
});
if (completionPreregistered && !durableReceiptReady) {
await retireFailedPersistence(
'background code result was blocked by content policy',
'definite',
);
}
logger.warn(
`[background] Generated code output for task ${task.id} was blocked by content policy.`,
);
return;
}
if (completionPreregistered && !durableReceiptReady) {
await retireFailedPersistence(
'background code result persistence failed',
'ambiguous',
);
}
logger.warn(
`[background] Failed to persist code result for task ${task.id}:`,
persistError,

View file

@ -40,6 +40,8 @@ export interface BackgroundToolResultState {
settledAt: Date;
/** This exact task owns a pre-registered automatic continuation delivery. */
completionWakeup?: true;
/** Independent receipts retain task-local delivery ownership. */
completionReceipt?: true;
resultClaim?: {
kind: 'manual' | 'wakeup';
claimId: string;

View file

@ -25,6 +25,7 @@ export * from './handlers';
export * from './guard';
export * from './harvest';
export * from './backgroundCompletion';
export * from './backgroundClaims';
export * from './backgroundCompletionWakeup';
export * from './initialize';
export * from './legacy';

View file

@ -589,6 +589,32 @@ describe('createAgentTriggerDeliveryEngine', () => {
expect(store.dead).toHaveBeenCalledWith(expect.objectContaining({ claimToken: 'claim-1' }));
});
it.each([false, true])(
'offers configured receipt recovery when already exhausted is %s',
async (exhausted) => {
const store = storeWith({
claimNext: jest.fn(async () => delivery({ attempts: exhausted ? 1 : 0 })),
});
const engine = createAgentTriggerDeliveryEngine(
{
store,
now: () => START,
dispatch: async () => {
throw new Error('database unavailable');
},
},
{ concurrency: 1, maxAttempts: 1, retryCapMs: 123_000 },
);
await engine.runTick();
expect(store.dead).toHaveBeenCalledWith(
expect.objectContaining({
receiptRetryAt: new Date(START.getTime() + 123_000),
error: expect.objectContaining({ retryable: true }),
}),
);
},
);
it('bounds a persisted last failure before exhausting its source', async () => {
const oversized = 'x'.repeat(3_000);
const store = storeWith({

View file

@ -183,6 +183,7 @@ export interface AgentTriggerDeliveryStore {
attempt: number;
error: AgentTriggerDeliveryFailure;
settledAt: Date;
receiptRetryAt?: Date;
}) => Promise<boolean>;
}
@ -409,6 +410,9 @@ export function createAgentTriggerDeliveryEngine(
attempt: delivery.attempts,
error: recorded,
settledAt: now(),
...(recorded.retryable && {
receiptRetryAt: new Date(now().getTime() + retryCapMs),
}),
});
if (deadLettered) {
logger.error('[agent-triggers] delivery dead-lettered after exhausting retries', {
@ -532,6 +536,9 @@ export function createAgentTriggerDeliveryEngine(
attempt,
error: recorded,
settledAt: attemptedAt,
...(recorded.retryable && {
receiptRetryAt: new Date(attemptedAt.getTime() + retryCapMs),
}),
});
if (deadLettered) {
logger.error('[agent-triggers] delivery dead-lettered', {

View file

@ -1,4 +1,5 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@ -180,6 +181,7 @@ describe('durable agent trigger service', () => {
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
expect.objectContaining({
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
@ -218,6 +220,7 @@ describe('durable agent trigger service', () => {
expect(methods.claimNextAgentTriggerDelivery).toHaveBeenCalledWith(
expect.objectContaining({
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
],

View file

@ -1,4 +1,5 @@
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@ -112,6 +113,9 @@ export interface AgentTriggerDeliveryPersistence {
completeAgentTriggerDelivery: AgentTriggerDeliveryStore['complete'];
retireAgentTriggerDelivery: AgentTriggerDeliveryMethods['retireAgentTriggerDelivery'];
renewAgentTriggerDeliveryProducerLease: AgentTriggerDeliveryMethods['renewAgentTriggerDeliveryProducerLease'];
persistAgentBackgroundToolResult?: AgentTriggerDeliveryMethods['persistAgentBackgroundToolResult'];
getAgentBackgroundToolResultClaim?: AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim'];
releaseAgentBackgroundToolResultClaims?: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
retryAgentTriggerDelivery: AgentTriggerDeliveryStore['retry'];
deadLetterAgentTriggerDelivery: AgentTriggerDeliveryStore['dead'];
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerStoredRecord | null>;
@ -168,6 +172,19 @@ export interface AgentTriggerService {
options?: { onlyIfUnclaimed?: boolean; onlyIfDead?: boolean },
) => Promise<boolean>;
renewProducerLease: (deliveryKey: string, sourceId: string, leaseUntil: Date) => Promise<boolean>;
persistBackgroundToolResult: (input: {
deliveryKey: string;
sourceId: string;
result: {
status: 'completed' | 'error' | 'cancelled';
output: string;
settledAt: Date;
};
}) => Promise<boolean>;
getBackgroundToolResultClaim: (
input: Parameters<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>[0],
) => ReturnType<AgentTriggerDeliveryMethods['getAgentBackgroundToolResultClaim']>;
releaseBackgroundToolResultClaims: AgentTriggerDeliveryMethods['releaseAgentBackgroundToolResultClaims'];
drainUser: (userId: string) => Promise<void>;
prepareUserPurge: (userId: string, fenceStartedAt: Date, tenantId?: string) => Promise<void>;
cancelUserPurge: (userId: string, fenceStartedAt: Date) => Promise<boolean>;
@ -183,6 +200,7 @@ function createDeliveryStore(
methods.claimNextAgentTriggerDelivery({
...input,
workerCapabilities: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
...(supportsDetachedActionCompletion()
@ -600,6 +618,21 @@ export function createAgentTriggerService(deps: AgentTriggerServiceDeps = {}): A
leaseUntil,
}),
),
persistBackgroundToolResult: (input) =>
runAsSystem(async () => {
const persist = requireMethods().persistAgentBackgroundToolResult;
return persist == null ? false : persist(input);
}),
getBackgroundToolResultClaim: (input) =>
runAsSystem(async () => {
const getClaim = requireMethods().getAgentBackgroundToolResultClaim;
return getClaim == null ? null : getClaim(input);
}),
releaseBackgroundToolResultClaims: (input) =>
runAsSystem(async () => {
const release = requireMethods().releaseAgentBackgroundToolResultClaims;
return release == null ? false : release(input);
}),
drainUser,
prepareUserPurge: (userId, fenceStartedAt, tenantId) =>
runAsSystem(async () =>

View file

@ -577,6 +577,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
});
});
@ -593,6 +594,7 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: false,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: false,
});
});
@ -609,9 +611,31 @@ describe('agent background task config', () => {
}
expect(result.data.endpoints?.agents?.backgroundTasks).toEqual({
completionWakeups: true,
completionResultMaxChars: 24 * 1024,
ordinaryToolCancellation: true,
});
});
it('accepts a bounded durable completion result limit', () => {
const result = configSchema.safeParse({
version: '1.0',
endpoints: { agents: { backgroundTasks: { completionResultMaxChars: 4096 } } },
});
expect(result.success).toBe(true);
if (result.success) {
expect(result.data.endpoints?.agents?.backgroundTasks?.completionResultMaxChars).toBe(4096);
}
});
it.each([0, 64 * 1024 + 1])('rejects an unsafe completion result limit: %s', (limit) => {
expect(
configSchema.safeParse({
version: '1.0',
endpoints: { agents: { backgroundTasks: { completionResultMaxChars: limit } } },
}).success,
).toBe(false);
});
});
describe('speechTab schema', () => {

View file

@ -22,6 +22,9 @@ import {
MIN_BALANCE_RESERVATION_TTL_MS,
DEFAULT_BALANCE_RESERVATION_TTL_MS,
} from './balance';
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT = 24 * 1024;
export const AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX = 64 * 1024;
import {
MAX_SUBAGENTS,
MAX_SUBAGENTS_CEILING,
@ -1465,6 +1468,15 @@ export const agentsEndpointSchema = baseEndpointSchema
backgroundTasks: z
.object({
completionWakeups: z.boolean().optional().default(true),
/** Maximum terminal output copied into the private completion receipt.
* Generated files remain governed by their separate attachment policy. */
completionResultMaxChars: z
.number()
.int()
.min(1)
.max(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_HARD_MAX)
.optional()
.default(AGENT_BACKGROUND_COMPLETION_RESULT_MAX_CHARS_DEFAULT),
/** Cooperative cancellation for process-local ordinary tools. Off
* by default so existing deployments opt into the new control. */
ordinaryToolCancellation: z.boolean().optional().default(false),

View file

@ -67,6 +67,8 @@ export {
isCompactionSemanticIndexProjection,
} from './types/compaction';
export {
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,

View file

@ -2230,10 +2230,14 @@ describe('Conversation Operations', () => {
const deleteAgentQueuedTurns = jest.fn(async () => {
transitions.push('queue-retired');
});
const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => {
transitions.push('trigger-results-erased');
});
const scopedMethods = createConversationMethods(mongoose, {
getMessages,
deleteMessages,
deleteAgentQueuedTurns,
eraseAgentTriggerDeliveryConversationResults,
});
await scopedMethods.deleteConvos(
@ -2249,7 +2253,99 @@ describe('Conversation Operations', () => {
expect(deleteAgentQueuedTurns).toHaveBeenCalledWith('user123', [
{ conversationId, tenantId: 'tenant-1' },
]);
expect(transitions).toEqual(['queue-retired', 'generation-drained']);
expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [
conversationId,
]);
expect(transitions).toEqual([
'queue-retired',
'generation-drained',
'trigger-results-erased',
]);
});
it('preserves background receipts when a pre-delete drain fails', async () => {
const conversationId = uuidv4();
await Conversation.create({
conversationId,
user: 'user123',
endpoint: EModelEndpoint.agents,
});
const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined);
const scopedMethods = createConversationMethods(mongoose, {
getMessages,
deleteMessages,
eraseAgentTriggerDeliveryConversationResults,
});
await expect(
scopedMethods.deleteConvos(
'user123',
{ conversationId },
{ beforeDelete: async () => Promise.reject(new Error('drain unavailable')) },
),
).rejects.toThrow('drain unavailable');
expect(await Conversation.findOne({ conversationId })).not.toBeNull();
expect(eraseAgentTriggerDeliveryConversationResults).not.toHaveBeenCalled();
});
it('finishes descendant bookkeeping and message cleanup when receipt erasure fails', async () => {
const parentId = uuidv4();
const childId = uuidv4();
const grandchildId = uuidv4();
const ids = [parentId, childId, grandchildId];
const project = await ChatProject.create({ user: 'user123', name: 'Receipt cleanup' });
await ConversationTag.create({ user: 'user123', tag: 'work', count: 3, position: 0 });
await Conversation.create(
ids.map((conversationId, depth) => ({
conversationId,
user: 'user123',
endpoint: EModelEndpoint.agents,
tags: ['work'],
chatProjectId: project._id.toString(),
...(depth === 0
? {}
: {
subagentThread: {
rootConversationId: parentId,
parentConversationId: ids[depth - 1],
parentMessageId: `message-${depth}`,
parentToolCallId: `call-${depth}`,
subagentType: 'agent-child',
subagentKind: 'agent',
depth,
},
}),
})),
);
const prepare = jest.fn(async () => undefined);
const erase = jest.fn(async () => {
throw new Error('receipt storage unavailable');
});
const scopedMethods = createConversationMethods(mongoose, {
getMessages,
deleteMessages,
prepareAgentTriggerConversationResultErasure: prepare,
eraseAgentTriggerDeliveryConversationResults: erase,
});
const result = await scopedMethods.deleteConvos('user123', { conversationId: parentId });
expect(result).toMatchObject({ deletedCount: 3, conversationIds: ids });
expect(prepare.mock.calls).toHaveLength(3);
expect(erase.mock.calls).toHaveLength(3);
expect(await Conversation.countDocuments({ user: 'user123' })).toBe(0);
expect((await ConversationTag.findOne({ user: 'user123', tag: 'work' }).lean())?.count).toBe(
0,
);
expect((await ChatProject.findById(project._id).lean())?.conversationCount).toBe(0);
expect(deleteMessages).toHaveBeenCalledWith({
user: 'user123',
conversationId: { $in: ids },
});
await expect(
scopedMethods.deleteConvos('user123', { conversationId: parentId }, { allowEmpty: true }),
).resolves.toMatchObject({ deletedCount: 0, conversationIds: [parentId] });
});
it('fails closed before deleting a conversation when queued-turn retirement fails', async () => {
@ -2468,13 +2564,22 @@ describe('Conversation Operations', () => {
});
it('allows a single-conversation cleanup retry after topology and messages are gone', async () => {
const result = await deleteConvos(
const eraseAgentTriggerDeliveryConversationResults = jest.fn(async () => undefined);
const scopedMethods = createConversationMethods(mongoose, {
getMessages,
deleteMessages,
eraseAgentTriggerDeliveryConversationResults,
});
const result = await scopedMethods.deleteConvos(
'user123',
{ conversationId: 'already-absent' },
{ allowEmpty: true },
);
expect(result.deletedCount).toBe(0);
expect(result.conversationIds).toEqual(['already-absent']);
expect(eraseAgentTriggerDeliveryConversationResults).toHaveBeenCalledWith('user123', [
'already-absent',
]);
});
it('supports an idempotent empty recovery sweep without hiding storage failures', async () => {

View file

@ -512,6 +512,14 @@ export interface ConversationMethodDeps
user: string,
conversations: Array<{ conversationId: string; tenantId?: string; allTenants?: true }>,
) => Promise<void>;
eraseAgentTriggerDeliveryConversationResults?: (
user: string,
conversationIds: string[],
) => Promise<void>;
prepareAgentTriggerConversationResultErasure?: (
user: string,
conversationIds: string[],
) => Promise<void>;
}
export function createConversationMethods(
@ -3235,7 +3243,18 @@ export function createConversationMethods(
})),
);
await options?.beforeDelete?.(waveIds);
await deps?.prepareAgentTriggerConversationResultErasure?.(user, waveIds);
const result = await Conversation.deleteMany({ user, conversationId: { $in: waveIds } });
if (result.deletedCount > 0) {
/** Result erasure is irreversible. Keep receipts intact when a
* pre-delete hook or the conversation delete itself fails, so a
* retained conversation cannot lose a receipt-only completion. */
try {
await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, waveIds);
} catch (error) {
logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error);
}
}
acknowledged &&= result.acknowledged;
deletedCount += result.deletedCount;
await reconcileDeletedWave(wave, result.deletedCount);
@ -3266,6 +3285,11 @@ export function createConversationMethods(
allTenants: true,
})),
);
try {
await deps?.eraseAgentTriggerDeliveryConversationResults?.(user, recoveryConversationIds);
} catch (error) {
logger.error('[deleteConvos] Receipt erasure deferred to durable cleanup', error);
}
}
const deleteConvoResult: DeleteResult = { acknowledged, deletedCount };

View file

@ -325,6 +325,10 @@ export function createMethods(
getMessages: messageMethods.getMessages,
deleteMessages: messageMethods.deleteMessages,
searchMessages: messageMethods.searchMessages,
eraseAgentTriggerDeliveryConversationResults:
agentTriggerDeliveryMethods.eraseAgentTriggerDeliveryConversationResults,
prepareAgentTriggerConversationResultErasure:
agentTriggerDeliveryMethods.prepareAgentTriggerConversationResultErasure,
deleteAgentQueuedTurns: async (user, conversations) => {
/** Queued-turn ownership is ObjectId-backed. Conversation methods also
* support synthetic/non-ObjectId owners in embedded integrations and

View file

@ -726,6 +726,7 @@ export interface MessageMethods {
cancelled?: true;
settledAt: Date;
completionWakeup?: true;
completionReceipt?: true;
resultClaim?: {
kind: 'manual' | 'wakeup';
claimId: string;
@ -1224,6 +1225,7 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
cancelled?: true;
settledAt: Date;
completionWakeup?: true;
completionReceipt?: true;
resultClaim?: {
kind: 'manual' | 'wakeup';
claimId: string;
@ -1274,6 +1276,9 @@ export function createMessageMethods(mongoose: typeof import('mongoose')): Messa
partPatch['content.$[part].tool_call.backgroundTask.cancelled'] =
backgroundTask.cancelled === true;
partPatch['content.$[part].tool_call.backgroundTask.settledAt'] = backgroundTask.settledAt;
if (backgroundTask.completionReceipt === true) {
partPatch['content.$[part].tool_call.backgroundTask.completionReceipt'] = true;
}
if (backgroundTask.completionWakeup === true) {
partPatch['content.$[part].tool_call.backgroundTask.completionWakeup'] = true;
} else {

View file

@ -18,12 +18,14 @@ import {
type AgentTriggerDeliveryMethods,
} from './triggerDelivery';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
} from '~/types/triggerDelivery';
import { createAgentTriggerLaneSequenceModel } from '../models/triggerLaneSequence';
import { createAgentTriggerUserPurgeModel } from '../models/triggerUserPurge';
import { createAgentTriggerDeliveryModel } from '../models/triggerDelivery';
import { createConversationModel } from '../models/convo';
import { createUserModel } from '../models/user';
jest.mock('~/config/winston', () => ({
@ -32,6 +34,7 @@ jest.mock('~/config/winston', () => ({
info: jest.fn(),
debug: jest.fn(),
}));
jest.mock('~/models/plugins/mongoMeili', () => jest.fn());
const DB_SETUP_TIMEOUT_MS = 60_000;
const START = new Date('2026-08-17T12:00:00.000Z');
@ -50,6 +53,7 @@ beforeAll(async () => {
LaneSequence = createAgentTriggerLaneSequenceModel(mongoose);
UserPurge = createAgentTriggerUserPurgeModel(mongoose);
User = createUserModel(mongoose);
createConversationModel(mongoose);
await Promise.all([Delivery.init(), LaneSequence.init(), UserPurge.init(), User.init()]);
methods = createAgentTriggerDeliveryMethods(mongoose);
}, DB_SETUP_TIMEOUT_MS);
@ -384,7 +388,7 @@ describe('agent trigger delivery methods', () => {
it('shields background completion work from workers that cannot resolve it', async () => {
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
const claimInput = {
@ -400,7 +404,7 @@ describe('agent trigger delivery methods', () => {
...claimInput,
workerId: 'background-capable-worker',
claimToken: 'background-capable-claim',
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1],
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
}),
).resolves.toMatchObject({ id: queued.delivery.id });
});
@ -412,7 +416,7 @@ describe('agent trigger delivery methods', () => {
enqueueInput({
deliveryKey: 'background-completion-producer-lease',
envelope: { event: { source } },
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
producerLeaseUntil: initialLease,
}),
);
@ -449,6 +453,323 @@ describe('agent trigger delivery methods', () => {
).resolves.toEqual({ status: 'live', leaseUntil: renewedUntil });
});
it('persists one private background result receipt independently of message rows', async () => {
const source = { id: 'background-tool-completion', type: 'internal' };
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
deliveryKey: 'background-completion-result-receipt',
envelope: { event: { source } },
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
const input = {
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
result: { status: 'completed' as const, output: 'durable output', settledAt: START },
};
await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true);
await expect(methods.persistAgentBackgroundToolResult(input)).resolves.toBe(true);
await expect(
methods.getAgentBackgroundToolResult({
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
}),
).resolves.toEqual(input.result);
await expect(
methods.persistAgentBackgroundToolResult({
...input,
result: { ...input.result, output: 'conflicting output' },
}),
).resolves.toBe(false);
const ordinaryRead = await Delivery.findById(queued.delivery.id).lean();
expect(ordinaryRead).not.toHaveProperty('backgroundToolResult');
});
it('erases private result receipts when their conversation is deleted', async () => {
const source = { id: 'background-tool-completion', type: 'internal' };
const user = new mongoose.Types.ObjectId();
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
deliveryKey: 'background-completion-result-to-erase',
user,
envelope: {
event: { source },
target: { conversationId: 'conversation-to-delete' },
},
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
await methods.persistAgentBackgroundToolResult({
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
result: { status: 'completed', output: 'private output', settledAt: START },
});
await methods.eraseAgentTriggerDeliveryConversationResults(user, ['conversation-to-delete']);
await expect(
methods.getAgentBackgroundToolResult({
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
}),
).resolves.toBeNull();
await expect(Delivery.exists({ _id: queued.delivery.id })).resolves.not.toBeNull();
await expect(
methods.persistAgentBackgroundToolResult({
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
result: { status: 'completed', output: 'late private output', settledAt: START },
}),
).resolves.toBe(false);
});
it('does not make legacy string-owned conversation cleanup depend on an ObjectId cast', async () => {
await expect(
methods.prepareAgentTriggerConversationResultErasure('legacy-user', ['conversation']),
).resolves.toBeUndefined();
await expect(
methods.eraseAgentTriggerDeliveryConversationResults('legacy-user', ['conversation']),
).resolves.toBeUndefined();
});
it('recovers receipt erasure after a descendant row was already deleted', async () => {
const user = new mongoose.Types.ObjectId();
const source = { id: 'background-tool-completion', type: 'internal' };
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
user,
envelope: { event: { source }, target: { conversationId: 'deleted-child' } },
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
const lookup = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id };
await methods.persistAgentBackgroundToolResult({
...lookup,
result: { status: 'completed', output: 'private child output', settledAt: START },
});
await methods.prepareAgentTriggerConversationResultErasure(user, ['deleted-child']);
await mongoose.models.Conversation.collection.insertOne({
user: user.toString(),
conversationId: 'deleted-child',
});
await methods.recoverAgentTriggerUserPurges();
await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.not.toBeNull();
await mongoose.models.Conversation.deleteOne({ user, conversationId: 'deleted-child' });
/** The conversation deletion committed, but the follow-up receipt erasure did not. */
await methods.recoverAgentTriggerUserPurges();
await expect(methods.getAgentBackgroundToolResult(lookup)).resolves.toBeNull();
await expect(
Delivery.exists({
_id: queued.delivery.id,
backgroundToolResultDeletionPendingAt: { $exists: true },
}),
).resolves.toBeNull();
await expect(
methods.persistAgentBackgroundToolResult({
...lookup,
result: { status: 'completed', output: 'late write', settledAt: START },
}),
).resolves.toBe(false);
});
it.each([
['receipt', 'BACKGROUND_TOOL_PRODUCER_LOST'],
['death', 'BACKGROUND_TOOL_PRODUCER_LOST'],
['receipt', 'PARENT_STATE_UNAVAILABLE'],
['death', 'PARENT_STATE_UNAVAILABLE'],
])('atomically fences result publication when %s wins before %s', async (winner, code) => {
const source = { id: 'background-tool-completion', type: 'internal' };
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
envelope: { event: { source } },
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
const owner = { workerId: 'worker', claimToken: 'claim', now: START };
const claimed = await methods.claimNextAgentTriggerDelivery({
...owner,
leaseUntil: new Date(START.getTime() + 60_000),
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
});
expect(claimed).not.toBeNull();
const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id: claimed!.id });
const write = () =>
methods.persistAgentBackgroundToolResult({
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
result: { status: 'completed', output: 'late output', settledAt: START },
});
const dead = () =>
methods.deadLetterAgentTriggerDelivery({
...owner,
id: claimed!.id,
attempt: attempt!,
settledAt: START,
error: transientFailure({ code, retryable: code !== 'BACKGROUND_TOOL_PRODUCER_LOST' }),
});
await expect(winner === 'receipt' ? write() : dead()).resolves.toBe(true);
await expect(winner === 'receipt' ? dead() : write()).resolves.toBe(false);
});
it.each(['shield', 'legacy'] as const)(
'recovers a committed receipt after generic retry exhaustion through the %s fence',
async (format) => {
const source = { id: 'background-tool-completion', type: 'internal' };
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
envelope: { event: { source } },
requiredWorkerCapability:
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
const owner = { workerId: 'worker', claimToken: 'claim', now: START };
const claimed = await methods.claimNextAgentTriggerDelivery({
...owner,
leaseUntil: new Date(START.getTime() + 60_000),
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
});
const id = claimed!.id;
const attempt = await methods.beginAgentTriggerDeliveryAttempt({ ...owner, id });
if (format !== 'shield') {
await Delivery.updateOne(
{ _id: id },
{
$set: {
status: 'capability_leased',
leaseBy: owner.workerId,
claimToken: owner.claimToken,
leaseUntil: new Date(START.getTime() + 60_000),
},
$unset: {
capabilityStatus: 1,
capabilityLeaseBy: 1,
capabilityLeaseUntil: 1,
capabilityClaimToken: 1,
},
},
);
}
const receipt = { deliveryKey: queued.delivery.deliveryKey, sourceId: source.id };
await methods.persistAgentBackgroundToolResult({
...receipt,
result: { status: 'completed', output: 'saved output', settledAt: START },
});
const receiptRetryAt = new Date(START.getTime() + 300_000);
const terminal = {
...owner,
id,
attempt: attempt!,
settledAt: START,
receiptRetryAt,
error: transientFailure({ code: 'PARENT_STATE_UNAVAILABLE' }),
};
await expect(
methods.deadLetterAgentTriggerDelivery({ ...terminal, claimToken: 'stale' }),
).resolves.toBe(false);
expect((await Delivery.findById(id).lean())?.attempts).toBe(attempt);
await expect(methods.deadLetterAgentTriggerDelivery(terminal)).resolves.toBe(false);
expect(await Delivery.findById(id).lean()).toMatchObject({
attempts: 0,
claimAvailableAt: receiptRetryAt,
...(format === 'shield'
? { capabilityStatus: 'pending' }
: { status: 'capability_pending' }),
});
await expect(methods.getAgentBackgroundToolResult(receipt)).resolves.toMatchObject({
output: 'saved output',
});
const recovered = await methods.claimNextAgentTriggerDelivery({
...owner,
claimToken: 'recovered',
now: receiptRetryAt,
leaseUntil: new Date(receiptRetryAt.getTime() + 60_000),
workerCapabilities: [AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2],
});
expect(recovered?.id).toBe(id);
expect(
await methods.beginAgentTriggerDeliveryAttempt({
...owner,
id,
claimToken: 'recovered',
now: receiptRetryAt,
}),
).toBe(1);
await expect(
methods.deadLetterAgentTriggerDelivery({
...terminal,
claimToken: 'recovered',
settledAt: receiptRetryAt,
error: transientFailure({ code: 'PARENT_NOT_FOUND', retryable: false }),
}),
).resolves.toBe(true);
},
);
it('claims an independent result receipt before returning its output', async () => {
const source = { id: 'background-tool-completion', type: 'internal' };
const user = new mongoose.Types.ObjectId();
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({
deliveryKey: 'background-completion-result-to-claim',
user,
envelope: {
event: {
source,
payload: { taskId: 'task-1', toolCallId: 'call-1', toolName: 'slow_tool' },
},
target: {
agentId: 'agent-1',
conversationId: 'conversation-1',
parentMessageId: 'response-1',
},
},
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
}),
);
await methods.persistAgentBackgroundToolResult({
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
result: { status: 'completed', output: 'private output', settledAt: START },
});
const claim = {
deliveryKey: queued.delivery.deliveryKey,
sourceId: source.id,
userId: user.toString(),
conversationId: 'conversation-1',
parentMessageId: 'response-1',
agentId: 'agent-1',
claimId: 'delivery-claim-1',
limit: 1,
};
await expect(methods.claimAgentBackgroundToolResults(claim)).resolves.toMatchObject({
status: 'acquired',
results: [{ taskId: 'task-1', output: 'private output' }],
});
await expect(
methods.claimAgentBackgroundToolResults({ ...claim, claimId: 'delivery-claim-2' }),
).resolves.toEqual({ status: 'claimed', claimId: 'delivery-claim-1' });
await expect(
methods.getAgentBackgroundToolResultClaim({
sourceId: source.id,
userId: user.toString(),
conversationId: 'conversation-1',
parentMessageId: 'response-1',
taskId: 'task-1',
}),
).resolves.toMatchObject({ kind: 'wakeup', claimId: 'delivery-claim-1' });
await expect(
methods.releaseAgentBackgroundToolResultClaims({
sourceId: source.id,
userId: user.toString(),
conversationId: 'conversation-1',
parentMessageId: 'response-1',
claimId: 'delivery-claim-1',
}),
).resolves.toBe(true);
});
it('keeps capability-fenced work limited to capable workers through lease recovery', async () => {
const queued = await methods.enqueueAgentTriggerDelivery(
enqueueInput({

View file

@ -1,6 +1,7 @@
import { createHash } from 'node:crypto';
import type { FilterQuery, Model, Types } from 'mongoose';
import type {
AgentBackgroundToolResultReceipt,
AgentEventActorDetachedAction,
AgentTriggerDeliveryClaim,
AgentEventActorReceipt,
@ -17,6 +18,8 @@ import type {
IAgentTriggerUserPurgeDocument,
} from '~/types/triggerDelivery';
import {
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1,
@ -129,6 +132,26 @@ export type AgentTriggerProducerLeaseStatus =
| { status: 'expired'; leaseUntil: Date }
| { status: 'missing' };
export interface PersistAgentBackgroundToolResultInput {
deliveryKey: string;
sourceId: string;
result: AgentBackgroundToolResultReceipt;
}
export type AgentBackgroundToolResultClaim =
| { status: 'not_ready' }
| { status: 'claimed'; claimId: string }
| {
status: 'acquired';
results: Array<{
taskId: string;
toolCallId: string;
toolName: string;
status: AgentBackgroundToolResultReceipt['status'];
output: string;
}>;
};
export interface AgentTriggerDeliveryFence {
id: string;
workerId: string;
@ -277,6 +300,37 @@ export interface AgentTriggerDeliveryMethods {
sourceId: string;
now: Date;
}) => Promise<AgentTriggerProducerLeaseStatus>;
persistAgentBackgroundToolResult: (
input: PersistAgentBackgroundToolResultInput,
) => Promise<boolean>;
getAgentBackgroundToolResult: (input: {
deliveryKey: string;
sourceId: string;
}) => Promise<AgentBackgroundToolResultReceipt | null>;
getAgentBackgroundToolResultClaim: (input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
taskId: string;
}) => Promise<AgentBackgroundToolResultReceipt['resultClaim'] | null>;
claimAgentBackgroundToolResults: (input: {
deliveryKey: string;
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
agentId?: string;
claimId: string;
limit?: number;
}) => Promise<AgentBackgroundToolResultClaim>;
releaseAgentBackgroundToolResultClaims: (input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
claimId: string;
}) => Promise<boolean>;
settleAgentTriggerHandlingOutcome: (
input: SettleAgentTriggerHandlingOutcomeInput,
) => Promise<boolean>;
@ -326,6 +380,7 @@ export interface AgentTriggerDeliveryMethods {
attempt: number;
error: AgentTriggerDeliveryFailure;
settledAt: Date;
receiptRetryAt?: Date;
},
) => Promise<boolean>;
getAgentTriggerDelivery: (deliveryKey: string) => Promise<AgentTriggerDeliveryRecord | null>;
@ -358,6 +413,14 @@ export interface AgentTriggerDeliveryMethods {
) => Promise<boolean>;
recoverAgentTriggerUserPurges: (limit?: number) => Promise<number>;
deleteAgentTriggerDeliveriesByUser: (user: string | Types.ObjectId) => Promise<void>;
eraseAgentTriggerDeliveryConversationResults: (
user: string | Types.ObjectId,
conversationIds: string[],
) => Promise<void>;
prepareAgentTriggerConversationResultErasure: (
user: string | Types.ObjectId,
conversationIds: string[],
) => Promise<void>;
}
function normalizeFailure(error: AgentTriggerDeliveryFailure): AgentTriggerDeliveryFailure {
@ -960,6 +1023,8 @@ export function createAgentTriggerDeliveryMethods(
if (
input.requiredWorkerCapability != null &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 &&
input.requiredWorkerCapability !==
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 &&
input.requiredWorkerCapability !== AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1
) {
@ -2122,7 +2187,12 @@ export function createAgentTriggerDeliveryMethods(
const renewed = await Delivery().updateOne(
{
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
requiredWorkerCapability: {
$in: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
],
},
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
status: {
@ -2161,7 +2231,12 @@ export function createAgentTriggerDeliveryMethods(
const delivery = await Delivery()
.findOne({
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
requiredWorkerCapability: {
$in: [
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1,
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
],
},
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
})
@ -2175,6 +2250,262 @@ export function createAgentTriggerDeliveryMethods(
: { status: 'expired', leaseUntil: delivery.producerLeaseUntil };
}
/** Stores terminal output on the pre-admitted delivery before attempting the
* parent-message projection. The first terminal receipt wins; exact retries
* are idempotent and conflicting rewrites fail closed. */
async function persistAgentBackgroundToolResult(
input: PersistAgentBackgroundToolResultInput,
): Promise<boolean> {
const { result } = input;
if (
input.deliveryKey.length === 0 ||
input.deliveryKey.length > 256 ||
input.sourceId.length === 0 ||
input.sourceId.length > 256 ||
!['completed', 'error', 'cancelled'].includes(result.status) ||
result.output.length > AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS ||
!(result.settledAt instanceof Date) ||
!Number.isFinite(result.settledAt.getTime())
) {
throw new TypeError('Invalid background tool result receipt');
}
const identity = {
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
backgroundToolResultErasedAt: { $exists: false },
};
const updated = await Delivery().updateOne(
{
...identity,
backgroundToolResult: { $exists: false },
status: { $nin: ['dead', 'capability_dead', 'succeeded'] },
capabilityStatus: { $ne: 'dead' },
},
{ $set: { backgroundToolResult: result } },
{ timestamps: false },
);
if (updated.matchedCount === 1) {
return true;
}
const existing = await Delivery()
.findOne(identity)
.select('+backgroundToolResult')
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
return (
existing?.backgroundToolResult?.status === result.status &&
existing.backgroundToolResult.output === result.output &&
existing.backgroundToolResult.settledAt.getTime() === result.settledAt.getTime()
);
}
async function getAgentBackgroundToolResult(input: {
deliveryKey: string;
sourceId: string;
}): Promise<AgentBackgroundToolResultReceipt | null> {
if (
input.deliveryKey.length === 0 ||
input.deliveryKey.length > 256 ||
input.sourceId.length === 0 ||
input.sourceId.length > 256
) {
throw new TypeError('Invalid background tool result receipt lookup');
}
const delivery = await Delivery()
.findOne({
deliveryKey: input.deliveryKey,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
})
.select('+backgroundToolResult')
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
return delivery?.backgroundToolResult ?? null;
}
async function getAgentBackgroundToolResultClaim(input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
taskId: string;
}): Promise<AgentBackgroundToolResultReceipt['resultClaim'] | null> {
const delivery = await Delivery()
.findOne({
...backgroundResultIdentity(input),
'envelope.event.payload.taskId': input.taskId,
'backgroundToolResult.resultClaim': { $exists: true },
})
.select('+backgroundToolResult')
.lean<Pick<IAgentTriggerDelivery, 'backgroundToolResult'>>();
return delivery?.backgroundToolResult?.resultClaim ?? null;
}
function backgroundResultIdentity(input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
agentId?: string;
}) {
return {
user: input.userId,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
'envelope.event.source.type': 'internal',
'envelope.event.source.id': input.sourceId,
'envelope.target.conversationId': input.conversationId,
'envelope.target.parentMessageId': input.parentMessageId,
...(input.agentId == null ? {} : { 'envelope.target.agentId': input.agentId }),
};
}
function projectBackgroundResult(delivery: {
envelope?: unknown;
backgroundToolResult?: AgentBackgroundToolResultReceipt;
}) {
const envelope = delivery.envelope;
if (envelope == null || typeof envelope !== 'object' || Array.isArray(envelope)) {
return null;
}
const event = (envelope as { event?: unknown }).event;
if (event == null || typeof event !== 'object' || Array.isArray(event)) {
return null;
}
const payload = (event as { payload?: unknown }).payload;
if (payload == null || typeof payload !== 'object' || Array.isArray(payload)) {
return null;
}
const { taskId, toolCallId, toolName } = payload as Record<string, unknown>;
const receipt = delivery.backgroundToolResult;
if (
receipt == null ||
typeof taskId !== 'string' ||
typeof toolCallId !== 'string' ||
typeof toolName !== 'string'
) {
return null;
}
return {
taskId,
toolCallId,
toolName,
status: receipt.status,
output: receipt.output,
};
}
/** Elects exactly one automatic continuation for an independently persisted
* result. A later stack layer may coalesce siblings after this ownership
* boundary, but the durable receipt itself is never dispatched unclaimed. */
async function claimAgentBackgroundToolResults(input: {
deliveryKey: string;
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
agentId?: string;
claimId: string;
limit?: number;
}): Promise<AgentBackgroundToolResultClaim> {
if (
input.deliveryKey.length === 0 ||
input.deliveryKey.length > 256 ||
input.sourceId.length === 0 ||
input.sourceId.length > 256 ||
input.userId.length === 0 ||
input.conversationId.length === 0 ||
input.conversationId.length > 256 ||
input.parentMessageId.length === 0 ||
input.parentMessageId.length > 256 ||
input.claimId.length === 0 ||
input.claimId.length > 128
) {
throw new TypeError('Invalid background tool result receipt claim');
}
const scope = backgroundResultIdentity(input);
const projection = 'envelope +backgroundToolResult';
let requested = await Delivery()
.findOne({ ...scope, deliveryKey: input.deliveryKey })
.select(projection)
.lean<Pick<IAgentTriggerDelivery, 'envelope' | 'backgroundToolResult'>>();
if (requested?.backgroundToolResult == null) {
return { status: 'not_ready' };
}
const existingClaim = requested.backgroundToolResult.resultClaim;
if (existingClaim != null && existingClaim.claimId !== input.claimId) {
return { status: 'claimed', claimId: existingClaim.claimId };
}
if (existingClaim == null) {
const claimed = await Delivery().updateOne(
{
...scope,
deliveryKey: input.deliveryKey,
backgroundToolResult: { $exists: true },
'backgroundToolResult.resultClaim': { $exists: false },
},
{
$set: {
'backgroundToolResult.resultClaim': {
kind: 'wakeup',
claimId: input.claimId,
claimedAt: new Date(),
},
},
},
{ timestamps: false },
);
if (claimed.modifiedCount !== 1) {
requested = await Delivery()
.findOne({ ...scope, deliveryKey: input.deliveryKey })
.select(projection)
.lean<Pick<IAgentTriggerDelivery, 'envelope' | 'backgroundToolResult'>>();
const winner = requested?.backgroundToolResult?.resultClaim;
if (winner?.claimId !== input.claimId) {
return winner == null
? { status: 'not_ready' }
: { status: 'claimed', claimId: winner.claimId };
}
}
}
const result = requested == null ? null : projectBackgroundResult(requested);
return result == null ? { status: 'not_ready' } : { status: 'acquired', results: [result] };
}
async function releaseAgentBackgroundToolResultClaims(input: {
sourceId: string;
userId: string;
conversationId: string;
parentMessageId: string;
claimId: string;
}): Promise<boolean> {
if (
input.sourceId.length === 0 ||
input.sourceId.length > 256 ||
input.userId.length === 0 ||
input.conversationId.length === 0 ||
input.conversationId.length > 256 ||
input.parentMessageId.length === 0 ||
input.parentMessageId.length > 256 ||
input.claimId.length === 0 ||
input.claimId.length > 128
) {
throw new TypeError('Invalid background tool result receipt claim release');
}
const scope = backgroundResultIdentity(input);
await Delivery().updateMany(
{ ...scope, 'backgroundToolResult.resultClaim.claimId': input.claimId },
{ $unset: { 'backgroundToolResult.resultClaim': 1 } },
{ timestamps: false },
);
return (
(await Delivery().exists({
...scope,
'backgroundToolResult.resultClaim.claimId': input.claimId,
})) == null
);
}
async function settleAgentTriggerHandlingOutcome(
input: SettleAgentTriggerHandlingOutcomeInput,
): Promise<boolean> {
@ -3076,10 +3407,26 @@ export function createAgentTriggerDeliveryMethods(
error: AgentTriggerDeliveryFailure;
availableAt: Date;
},
): Promise<boolean> {
return retryDelivery(input);
}
async function retryDelivery(
input: Parameters<AgentTriggerDeliveryMethods['retryAgentTriggerDelivery']>[0],
recoverReceipt = false,
): Promise<boolean> {
const error = normalizeFailure(input.error);
const receiptFence = recoverReceipt
? {
requiredWorkerCapability:
AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
backgroundToolResult: { $exists: true },
}
: {};
const recoveredAttempt = recoverReceipt ? { attempts: 0 } : {};
const retryUpdate = (status: 'pending' | 'capability_pending') => ({
$set: {
...recoveredAttempt,
status,
availableAt: input.availableAt,
claimAvailableAt: input.availableAt,
@ -3108,9 +3455,10 @@ export function createAgentTriggerDeliveryMethods(
},
});
const shieldResult = await Delivery().updateOne(
{ _id: input.id, ...shieldCapabilityFence(input) },
{ _id: input.id, ...shieldCapabilityFence(input), ...receiptFence },
{
$set: {
...recoveredAttempt,
status: 'leased',
availableAt: input.availableAt,
capabilityStatus: 'pending',
@ -3134,14 +3482,14 @@ export function createAgentTriggerDeliveryMethods(
return true;
}
const capabilityResult = await Delivery().updateOne(
{ _id: input.id, ...legacyCapabilityFence(input) },
{ _id: input.id, ...legacyCapabilityFence(input), ...receiptFence },
retryUpdate('capability_pending'),
);
if (capabilityResult.modifiedCount === 1) {
return true;
}
const result = await Delivery().updateOne(
{ _id: input.id, ...ordinaryFence(input) },
{ _id: input.id, ...ordinaryFence(input), ...receiptFence },
retryUpdate('pending'),
);
return result.modifiedCount === 1;
@ -3152,9 +3500,23 @@ export function createAgentTriggerDeliveryMethods(
attempt: number;
error: AgentTriggerDeliveryFailure;
settledAt: Date;
receiptRetryAt?: Date;
},
): Promise<boolean> {
const error = normalizeFailure(input.error);
if (
error.retryable &&
input.receiptRetryAt != null &&
(await retryDelivery({ ...input, availableAt: input.receiptRetryAt }, true))
) {
return false;
}
/** Publication must contend with every retryable terminal transition,
* including a receipt committed between recovery and the dead-letter CAS. */
const receiptFence =
error.retryable || error.code === 'BACKGROUND_TOOL_PRODUCER_LOST'
? { backgroundToolResult: { $exists: false } }
: {};
const deadUpdate = (status: 'dead' | 'capability_dead') => ({
$set: { status, settledAt: input.settledAt, lastError: error },
$unset: { leaseBy: 1, leaseUntil: 1, claimToken: 1, expiresAt: 1 },
@ -3174,7 +3536,7 @@ export function createAgentTriggerDeliveryMethods(
},
});
const shieldDead = await Delivery().updateOne(
{ _id: input.id, ...shieldCapabilityFence(input) },
{ _id: input.id, ...shieldCapabilityFence(input), ...receiptFence },
{
$set: {
// The pre-capability runtime already treats `capability_dead` as a
@ -3201,16 +3563,20 @@ export function createAgentTriggerDeliveryMethods(
return true;
}
const capabilityDead = await Delivery().updateOne(
{ _id: input.id, ...legacyCapabilityFence(input) },
{ _id: input.id, ...legacyCapabilityFence(input), ...receiptFence },
deadUpdate('capability_dead'),
);
if (capabilityDead.modifiedCount === 1) {
return true;
}
const dead = await Delivery()
.findOneAndUpdate({ _id: input.id, ...ordinaryFence(input) }, deadUpdate('dead'), {
new: true,
})
.findOneAndUpdate(
{ _id: input.id, ...ordinaryFence(input), ...receiptFence },
deadUpdate('dead'),
{
new: true,
},
)
.select('_id orderingKey batchMemberIds batchMembersSettledAt requeueCount')
.lean<
Pick<
@ -3512,6 +3878,31 @@ export function createAgentTriggerDeliveryMethods(
if (!Number.isSafeInteger(limit) || limit <= 0) {
throw new TypeError('Agent trigger purge recovery limit must be a positive integer');
}
const pending = await Delivery()
.find({
backgroundToolResultDeletionPendingAt: { $exists: true },
})
.select('user envelope')
.sort({ backgroundToolResultDeletionPendingAt: 1, _id: 1 })
.limit(Math.min(limit, MAX_PURGE_RECOVERY_LIMIT))
.lean<IAgentTriggerDelivery[]>();
for (const row of pending) {
const conversationId = (row.envelope as { target?: { conversationId?: string } }).target
?.conversationId;
if (conversationId == null) {
continue;
}
const exists = await mongoose.models.Conversation.exists({ user: row.user, conversationId });
if (exists == null) {
await eraseAgentTriggerDeliveryConversationResults(row.user, [conversationId]);
} else {
await Delivery().updateOne(
{ _id: row._id, backgroundToolResultDeletionPendingAt: { $exists: true } },
{ $currentDate: { backgroundToolResultDeletionPendingAt: true } },
{ timestamps: false },
);
}
}
const markers = await UserPurge()
.find({})
.sort({ updatedAt: 1, _id: 1 })
@ -3565,6 +3956,44 @@ export function createAgentTriggerDeliveryMethods(
await UserPurge().deleteOne({ _id: user });
}
async function eraseAgentTriggerDeliveryConversationResults(
user: string | Types.ObjectId,
conversationIds: string[],
): Promise<void> {
if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) {
return;
}
const erasedAt = new Date();
await Delivery().updateMany(
{
user,
'envelope.target.conversationId': { $in: conversationIds },
},
{
$set: { backgroundToolResultErasedAt: erasedAt },
$unset: { backgroundToolResult: 1, backgroundToolResultDeletionPendingAt: 1 },
},
{ timestamps: false },
);
}
async function prepareAgentTriggerConversationResultErasure(
user: string | Types.ObjectId,
conversationIds: string[],
): Promise<void> {
if (conversationIds.length === 0 || !mongoose.isObjectIdOrHexString(user)) return;
await Delivery().updateMany(
{
user,
requiredWorkerCapability: AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2,
'envelope.target.conversationId': { $in: conversationIds },
backgroundToolResultErasedAt: { $exists: false },
},
{ $set: { backgroundToolResultDeletionPendingAt: new Date() } },
{ timestamps: false },
);
}
return {
ensureAgentTriggerDeliveryIndexes,
enqueueAgentTriggerDelivery,
@ -3578,6 +4007,11 @@ export function createAgentTriggerDeliveryMethods(
retireAgentTriggerDelivery,
renewAgentTriggerDeliveryProducerLease,
getAgentTriggerDeliveryProducerLease,
persistAgentBackgroundToolResult,
getAgentBackgroundToolResult,
getAgentBackgroundToolResultClaim,
claimAgentBackgroundToolResults,
releaseAgentBackgroundToolResultClaims,
settleAgentTriggerHandlingOutcome,
admitAgentEventActorAction,
releaseAgentEventActorAction,
@ -3606,5 +4040,7 @@ export function createAgentTriggerDeliveryMethods(
cancelAgentTriggerUserPurge,
recoverAgentTriggerUserPurges,
deleteAgentTriggerDeliveriesByUser,
eraseAgentTriggerDeliveryConversationResults,
prepareAgentTriggerConversationResultErasure,
};
}

View file

@ -1,5 +1,6 @@
import { Schema } from 'mongoose';
import type { IAgentTriggerDeliveryDocument } from '~/types/triggerDelivery';
import { AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS } from '~/types/triggerDelivery';
const failureSchema = new Schema(
{
@ -147,6 +148,38 @@ const triggerDeliverySchema: Schema<IAgentTriggerDeliveryDocument> = new Schema(
capabilityClaimToken: { type: String },
/** Private process-owner heartbeat; never projected to legacy consumers. */
producerLeaseUntil: { type: Date, select: false },
backgroundToolResult: {
type: new Schema(
{
status: {
type: String,
enum: ['completed', 'error', 'cancelled'],
required: true,
},
output: {
type: String,
required: true,
maxlength: AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS,
},
settledAt: { type: Date, required: true },
resultClaim: {
type: new Schema(
{
kind: { type: String, enum: ['wakeup'], required: true },
claimId: { type: String, required: true, maxlength: 128 },
claimedAt: { type: Date, required: true },
},
{ _id: false },
),
required: false,
},
},
{ _id: false },
),
select: false,
},
backgroundToolResultErasedAt: { type: Date, select: false },
backgroundToolResultDeletionPendingAt: { type: Date, select: false },
attempts: { type: Number, required: true, default: 0, min: 0 },
availableAt: { type: Date, required: true },
envelopeBytes: { type: Number, min: 0 },
@ -224,6 +257,7 @@ triggerDeliverySchema.index({ 'actorReceipt.resolution': 1 }, { sparse: true });
triggerDeliverySchema.index({ user: 1, actorActionAdmittedAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ stagingRecoveryAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ laneCleanupPendingAt: 1 }, { sparse: true });
triggerDeliverySchema.index({ backgroundToolResultDeletionPendingAt: 1 }, { sparse: true });
// Only successful rows receive expiresAt. Dead letters remain available until
// an operator explicitly requeues or removes them.
triggerDeliverySchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });

View file

@ -15,7 +15,10 @@ export type AgentTriggerDeliveryStatus =
export const AGENT_TRIGGER_WORKER_CAPABILITY_DETACHED_ACTION_V1 = 'event_actor_detached_action_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_V1 =
'background_tool_completion_v1';
export const AGENT_TRIGGER_WORKER_CAPABILITY_BACKGROUND_COMPLETION_RECEIPT_V2 =
'background_tool_completion_receipt_v2';
export const AGENT_TRIGGER_WORKER_CAPABILITY_QUEUED_TURN_V1 = 'agent_queued_turn_v1';
export const AGENT_BACKGROUND_TOOL_RESULT_STORAGE_MAX_CHARS: number = 64 * 1024;
export type AgentTriggerDeliveryOutcome = 'succeeded' | 'retry' | 'dead';
export interface AgentTriggerHandlingState {
@ -77,6 +80,20 @@ export interface AgentTriggerDeliveryFailure {
status?: number;
}
/** Private terminal result produced before a background-completion delivery is
* resolved. It is independent of the parent message projection because a fast
* task can settle before that response row exists. */
export interface AgentBackgroundToolResultReceipt {
status: 'completed' | 'error' | 'cancelled';
output: string;
settledAt: Date;
resultClaim?: {
kind: 'wakeup';
claimId: string;
claimedAt: Date;
};
}
export interface AgentTriggerDeliveryHistoryEntry {
attempt: number;
outcome: AgentTriggerDeliveryOutcome;
@ -108,6 +125,11 @@ export interface IAgentTriggerDelivery {
capabilityClaimToken?: string;
/** Durable liveness evidence for process-owned capability work. */
producerLeaseUntil?: Date;
/** Durable source of truth for a background completion. */
backgroundToolResult?: AgentBackgroundToolResultReceipt;
/** Conversation-deletion fence preventing a late producer from restoring private output. */
backgroundToolResultErasedAt?: Date;
backgroundToolResultDeletionPendingAt?: Date;
attempts: number;
availableAt: Date;
envelopeBytes?: number;